hilos-agent 0.10.1 → 0.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +54 -10
- package/bin/hilos-agent.mjs +87 -14
- package/package.json +7 -5
- package/src/acp-session.mjs +16 -1
- package/src/cli.mjs +34 -1
- package/src/codex-mcp-session.mjs +36 -0
- package/src/config.mjs +39 -5
- package/src/conflicts.mjs +156 -0
- package/src/cursor-store.mjs +43 -8
- package/src/handler.mjs +1197 -127
- package/src/iterate-claim-recovery.mjs +574 -0
- package/src/local-harness-events.mjs +370 -0
- package/src/local-harness-transport.mjs +157 -0
- package/src/mcp.mjs +411 -34
- package/src/model-resolve.mjs +3 -1
- package/src/opencode-session.mjs +22 -2
- package/src/progress-emitter.mjs +1 -1
- package/src/queue.mjs +70 -3
- package/src/redact.mjs +360 -7
- package/src/run-inputs.mjs +211 -0
- package/src/run.mjs +462 -76
- package/src/thread-pr.mjs +26 -4
- package/src/webmcp-bridge.mjs +77 -4
package/src/run.mjs
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
import { hostname } from "node:os";
|
|
6
6
|
import { basename } from "node:path";
|
|
7
7
|
import { makeClient } from "./mcp.mjs";
|
|
8
|
-
import {
|
|
8
|
+
import { loadMentionCursorState, saveMentionCursor } from "./cursor-store.mjs";
|
|
9
9
|
import { mentionHandle } from "./daemon.mjs";
|
|
10
10
|
import { handleTask } from "./handler.mjs";
|
|
11
11
|
import { createQueue, looksLikeCancel, dedupeKey } from "./queue.mjs";
|
|
@@ -15,6 +15,12 @@ import { startWake, createWakeGate } from "./wake.mjs";
|
|
|
15
15
|
import { scanReplyBridge, handleReplyBridgeJob } from "./reply-bridge.mjs";
|
|
16
16
|
import { detectVendor, fastChatCmd, webCapability } from "./progress-emitter.mjs";
|
|
17
17
|
import { commandArgv } from "./argv.mjs";
|
|
18
|
+
import {
|
|
19
|
+
createIterateClaimRecoveryStore,
|
|
20
|
+
reconcileDaemonIterateClaims,
|
|
21
|
+
} from "./iterate-claim-recovery.mjs";
|
|
22
|
+
|
|
23
|
+
const MAX_IMMEDIATE_SEQUENCE_PAGES = 8;
|
|
18
24
|
|
|
19
25
|
/**
|
|
20
26
|
* The poll loop. Embeddable: pass a `signal` to stop it cleanly (interrupts the
|
|
@@ -28,9 +34,28 @@ import { commandArgv } from "./argv.mjs";
|
|
|
28
34
|
* log?: any,
|
|
29
35
|
* signal?: AbortSignal,
|
|
30
36
|
* onEvent?: (event: { type: string, [k: string]: any }) => void,
|
|
37
|
+
* mentionCursorDir?: string,
|
|
38
|
+
* iterateClaimRecoveryDir?: string,
|
|
39
|
+
* iterateClaimRecoveryRetryDelay?: () => Promise<void>,
|
|
40
|
+
* iterateClaimShutdownReconcileAttempts?: number,
|
|
41
|
+
* iterateClaimShutdownReconcileTimeoutMs?: number,
|
|
31
42
|
* }} [opts]
|
|
32
43
|
*/
|
|
33
|
-
export async function run(
|
|
44
|
+
export async function run(
|
|
45
|
+
cfg,
|
|
46
|
+
{
|
|
47
|
+
handler = handleTask,
|
|
48
|
+
log = console,
|
|
49
|
+
signal,
|
|
50
|
+
onEvent,
|
|
51
|
+
mentionCursorDir,
|
|
52
|
+
iterateClaimRecoveryDir,
|
|
53
|
+
iterateClaimRecoveryRetryDelay = () =>
|
|
54
|
+
new Promise((resolve) => setTimeout(resolve, 1000)),
|
|
55
|
+
iterateClaimShutdownReconcileAttempts = 2,
|
|
56
|
+
iterateClaimShutdownReconcileTimeoutMs = 1000,
|
|
57
|
+
} = {},
|
|
58
|
+
) {
|
|
34
59
|
if (!cfg.token) {
|
|
35
60
|
// Throw, don't process.exit — the CLI's main().catch prints + exits 1 exactly
|
|
36
61
|
// as before, and an embedding host (the desktop Local Agents runner) must
|
|
@@ -48,28 +73,93 @@ export async function run(cfg, { handler = handleTask, log = console, signal, on
|
|
|
48
73
|
/* a bad event listener must never break the daemon */
|
|
49
74
|
}
|
|
50
75
|
};
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
const { ping: onWakePing, sleep: interruptibleSleep } = createWakeGate({ signal });
|
|
76
|
+
let stoppingEmitted = false;
|
|
77
|
+
const emitStopping = () => {
|
|
78
|
+
if (stoppingEmitted) return;
|
|
79
|
+
stoppingEmitted = true;
|
|
80
|
+
emit({ type: "status", text: "stopping" });
|
|
81
|
+
};
|
|
58
82
|
|
|
59
83
|
// Already asked to stop before we even connect → return cleanly, do nothing.
|
|
60
84
|
if (signal?.aborted) {
|
|
61
|
-
|
|
85
|
+
emitStopping();
|
|
62
86
|
return;
|
|
63
87
|
}
|
|
88
|
+
const runController = new AbortController();
|
|
89
|
+
const abortFromCaller = () => runController.abort(signal?.reason);
|
|
90
|
+
signal?.addEventListener("abort", abortFromCaller, { once: true });
|
|
91
|
+
const runSignal = runController.signal;
|
|
92
|
+
// The wake doorbell (0824): a Realtime ping resolves the pending inter-poll
|
|
93
|
+
// sleep so the next list_mentions poll happens NOW. A ping that lands while a
|
|
94
|
+
// poll is still running is remembered, so the following sleep returns
|
|
95
|
+
// immediately instead of the ping being lost to the race. That race lives in
|
|
96
|
+
// createWakeGate (wake.mjs) so it can be unit-tested; the sleep it hands back
|
|
97
|
+
// also resolves on abort, so caller cancellation or a terminal auth response
|
|
98
|
+
// never waits out the poll interval.
|
|
99
|
+
const { ping: onWakePing, sleep: interruptibleSleep } = createWakeGate({
|
|
100
|
+
signal: runSignal,
|
|
101
|
+
});
|
|
102
|
+
const canceledByRunSignal = (error) =>
|
|
103
|
+
Boolean(signal?.aborted && (error?.abortedByCaller || error?.name === "AbortError"));
|
|
104
|
+
// Match the typed client error by its stable cross-realm name. This also
|
|
105
|
+
// works for embedding hosts and test doubles that do not share a constructor.
|
|
106
|
+
const isAuthStop = (error) => error?.name === "McpAuthStopError";
|
|
107
|
+
const reportAuthStop = (error) => {
|
|
108
|
+
if (!isAuthStop(error)) return false;
|
|
109
|
+
emitStopping();
|
|
110
|
+
return true;
|
|
111
|
+
};
|
|
64
112
|
|
|
65
|
-
const
|
|
113
|
+
const client = makeClient({ url: cfg.url, token: cfg.token });
|
|
114
|
+
// Keep injected/older test clients compatible; the real makeClient always
|
|
115
|
+
// provides this lifecycle signal and terminal auth still latches through it.
|
|
116
|
+
const { tool, listTools, listToolNames } = client;
|
|
117
|
+
const stopSignal = client.stopSignal ?? new AbortController().signal;
|
|
118
|
+
const abortFromAuthStop = () => runController.abort(stopSignal.reason);
|
|
119
|
+
stopSignal.addEventListener("abort", abortFromAuthStop, { once: true });
|
|
120
|
+
const detachLifecycleSources = () => {
|
|
121
|
+
signal?.removeEventListener("abort", abortFromCaller);
|
|
122
|
+
stopSignal.removeEventListener("abort", abortFromAuthStop);
|
|
123
|
+
};
|
|
66
124
|
|
|
67
|
-
|
|
125
|
+
let who;
|
|
126
|
+
try {
|
|
127
|
+
who = await tool("whoami", {}, { signal: runSignal });
|
|
128
|
+
} catch (error) {
|
|
129
|
+
detachLifecycleSources();
|
|
130
|
+
if (reportAuthStop(error)) throw error;
|
|
131
|
+
if (canceledByRunSignal(error)) {
|
|
132
|
+
emitStopping();
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
throw error;
|
|
136
|
+
}
|
|
68
137
|
const me = { ...who, handle: mentionHandle(who.agentName) };
|
|
69
138
|
if (!me.handle) {
|
|
139
|
+
detachLifecycleSources();
|
|
70
140
|
// Throw for the same reason as the token guard above — embed-safe, CLI-equal.
|
|
71
141
|
throw new Error("This agent has no display name / handle — set one in hilos, then reconnect.");
|
|
72
142
|
}
|
|
143
|
+
// Initialized only after the server advertises the complete recovery
|
|
144
|
+
// protocol below. Older servers keep their pre-recovery Iterate behavior.
|
|
145
|
+
let iterateClaimRecoveryStore = null;
|
|
146
|
+
let queue = null;
|
|
147
|
+
let wake = null;
|
|
148
|
+
let reconcileClaimsOnce = async () => true;
|
|
149
|
+
const withinShutdownReconcileBound = async (promise) => {
|
|
150
|
+
let timer;
|
|
151
|
+
try {
|
|
152
|
+
return await Promise.race([
|
|
153
|
+
Promise.resolve(promise),
|
|
154
|
+
new Promise((resolve) => {
|
|
155
|
+
timer = setTimeout(() => resolve(false), iterateClaimShutdownReconcileTimeoutMs);
|
|
156
|
+
}),
|
|
157
|
+
]);
|
|
158
|
+
} finally {
|
|
159
|
+
if (timer) clearTimeout(timer);
|
|
160
|
+
}
|
|
161
|
+
};
|
|
162
|
+
try {
|
|
73
163
|
log.log(`hilos-agent: ${me.agentName} (@${me.handle}) — ${cfg.url}`);
|
|
74
164
|
if (cfg.channelId) log.log(`scope: channel ${cfg.channelId}`);
|
|
75
165
|
log.log(`repos: ${Object.keys(cfg.repos).join(", ") || "(none configured)"}`);
|
|
@@ -94,11 +184,11 @@ export async function run(cfg, { handler = handleTask, log = console, signal, on
|
|
|
94
184
|
// listTools (schemas included) is the primary read; a client exposing only
|
|
95
185
|
// listToolNames (an embedder's minimal mock, or an older embedded client)
|
|
96
186
|
// still names the tools — it just never advertises arg-level capabilities.
|
|
97
|
-
const toolList = typeof listTools === "function" ? await listTools() : [];
|
|
187
|
+
const toolList = typeof listTools === "function" ? await listTools({ signal: runSignal }) : [];
|
|
98
188
|
const toolNames = toolList.length
|
|
99
189
|
? toolList.map((t) => t.name)
|
|
100
190
|
: typeof listToolNames === "function"
|
|
101
|
-
? await listToolNames()
|
|
191
|
+
? await listToolNames({ signal: runSignal })
|
|
102
192
|
: [];
|
|
103
193
|
const useMentions = toolNames.includes("list_mentions");
|
|
104
194
|
// Arg-level capability sniffing (0866): a long-poll-capable server declares
|
|
@@ -107,7 +197,45 @@ export async function run(cfg, { handler = handleTask, log = console, signal, on
|
|
|
107
197
|
const toolArgProps = (name) =>
|
|
108
198
|
toolList.find((t) => t.name === name)?.inputSchema?.properties ?? {};
|
|
109
199
|
const serverMentionWait = "waitMs" in toolArgProps("list_mentions");
|
|
200
|
+
const serverMentionSequence = "sinceMentionSeq" in toolArgProps("list_mentions");
|
|
201
|
+
const serverMentionSequenceBootstrap = "initializeMentionSeq" in toolArgProps("list_mentions");
|
|
110
202
|
const serverDecisionWait = "waitMs" in toolArgProps("get_permission_decision");
|
|
203
|
+
const updateRunArgs = toolArgProps("update_run");
|
|
204
|
+
const serverIterateClaimRecovery =
|
|
205
|
+
"claimId" in updateRunArgs &&
|
|
206
|
+
"confirmIterateClaim" in updateRunArgs &&
|
|
207
|
+
"releaseIterateClaim" in updateRunArgs;
|
|
208
|
+
if (me.agentId && serverIterateClaimRecovery) {
|
|
209
|
+
iterateClaimRecoveryStore = createIterateClaimRecoveryStore({
|
|
210
|
+
agentId: me.agentId,
|
|
211
|
+
url: cfg.url,
|
|
212
|
+
token: cfg.token,
|
|
213
|
+
...(iterateClaimRecoveryDir ? { dir: iterateClaimRecoveryDir } : {}),
|
|
214
|
+
log,
|
|
215
|
+
});
|
|
216
|
+
if (!iterateClaimRecoveryStore.acquireLock()) {
|
|
217
|
+
throw new Error("Another local daemon already owns this agent and server connection.");
|
|
218
|
+
}
|
|
219
|
+
reconcileClaimsOnce = async (context) => {
|
|
220
|
+
try {
|
|
221
|
+
const recovery = await reconcileDaemonIterateClaims({
|
|
222
|
+
tool,
|
|
223
|
+
recoveryStore: iterateClaimRecoveryStore,
|
|
224
|
+
log,
|
|
225
|
+
});
|
|
226
|
+
if (recovery.unresolved === 0) return true;
|
|
227
|
+
log.error(
|
|
228
|
+
`iterate claim recovery unresolved (${recovery.unresolved}) ${context}; work remains paused`,
|
|
229
|
+
);
|
|
230
|
+
return false;
|
|
231
|
+
} catch (error) {
|
|
232
|
+
log.error(
|
|
233
|
+
`iterate claim recovery failed ${context}: ${error?.message || error}; work remains paused`,
|
|
234
|
+
);
|
|
235
|
+
return false;
|
|
236
|
+
}
|
|
237
|
+
};
|
|
238
|
+
}
|
|
111
239
|
// Capabilities of THIS server, so the handler degrades gracefully on older
|
|
112
240
|
// deploys (e.g. no edit_message → no live heartbeat, rather than erroring).
|
|
113
241
|
const caps = {
|
|
@@ -142,6 +270,15 @@ export async function run(cfg, { handler = handleTask, log = console, signal, on
|
|
|
142
270
|
// server has to offer the tool, and the operator has to have turned
|
|
143
271
|
// `uploadTranscripts` on. Older servers simply never see the call.
|
|
144
272
|
uploadTranscript: toolNames.includes("upload_run_transcript"),
|
|
273
|
+
// Runtime-neutral replay exhaust (1148). The daemon only projects local
|
|
274
|
+
// coding streams when this exact ingest boundary exists; older servers
|
|
275
|
+
// keep the current progress/transcript behavior byte-for-byte.
|
|
276
|
+
hepEvents: toolNames.includes("append_run_events"),
|
|
277
|
+
// Room directions for a running job (1181/1182). When the server can take
|
|
278
|
+
// the receipt, the daemon declares `next_turn` at start_run, reads
|
|
279
|
+
// `pendingInputs` off its heartbeat, and runs one more turn with them.
|
|
280
|
+
// Older servers never list a direction, and the daemon never declares.
|
|
281
|
+
runInputs: toolNames.includes("acknowledge_run_input"),
|
|
145
282
|
// Emoji reactions (0860): the lightest answer to an untagged DM message
|
|
146
283
|
// that needs no words. Absent on older servers → the agent stays quiet
|
|
147
284
|
// instead, exactly as if it had no hand to wave.
|
|
@@ -156,7 +293,6 @@ export async function run(cfg, { handler = handleTask, log = console, signal, on
|
|
|
156
293
|
// On an older server the tool is absent and the daemon polls exactly as
|
|
157
294
|
// before; with it, a new message in the workspace rings a content-free
|
|
158
295
|
// Realtime broadcast that cuts mention pickup from pollMs to instant.
|
|
159
|
-
let wake = null;
|
|
160
296
|
if (toolNames.includes("get_wake_channel")) {
|
|
161
297
|
try {
|
|
162
298
|
const chan = await tool("get_wake_channel");
|
|
@@ -181,6 +317,7 @@ export async function run(cfg, { handler = handleTask, log = console, signal, on
|
|
|
181
317
|
log.log("wake: realtime doorbell enabled (polling stays as fallback)");
|
|
182
318
|
}
|
|
183
319
|
} catch (e) {
|
|
320
|
+
if (isAuthStop(e)) throw e;
|
|
184
321
|
log.log(`wake: unavailable (${e.message}) — polling only`);
|
|
185
322
|
}
|
|
186
323
|
}
|
|
@@ -214,6 +351,7 @@ export async function run(cfg, { handler = handleTask, log = console, signal, on
|
|
|
214
351
|
});
|
|
215
352
|
}
|
|
216
353
|
} catch (e) {
|
|
354
|
+
if (isAuthStop(e)) throw e;
|
|
217
355
|
log.error(`link_folder failed for ${path}: ${e?.message || e}`);
|
|
218
356
|
}
|
|
219
357
|
}
|
|
@@ -231,17 +369,102 @@ export async function run(cfg, { handler = handleTask, log = console, signal, on
|
|
|
231
369
|
// Resume from the persisted cursor (0866) so mentions that arrived while the
|
|
232
370
|
// daemon was down are delivered on restart instead of silently skipped —
|
|
233
371
|
// bounded by catchupMs. `backfill` still means "from the beginning" and wins.
|
|
234
|
-
const
|
|
372
|
+
const persistedCursorState = cfg.backfill
|
|
235
373
|
? null
|
|
236
|
-
:
|
|
374
|
+
: loadMentionCursorState(me.agentId, {
|
|
375
|
+
catchupMs: cfg.catchupMs,
|
|
376
|
+
...(mentionCursorDir ? { dir: mentionCursorDir } : {}),
|
|
377
|
+
});
|
|
378
|
+
const persistedCursor = persistedCursorState?.createdAt ?? null;
|
|
237
379
|
if (persistedCursor) log.log(`resuming mention cursor from ${persistedCursor}`);
|
|
238
380
|
const cursor = {
|
|
239
381
|
value: persistedCursor ?? (since ? new Date(since).toISOString() : null),
|
|
382
|
+
seq: serverMentionSequence ? persistedCursorState?.mentionSeq ?? null : null,
|
|
240
383
|
};
|
|
384
|
+
let mentionSequenceActive = serverMentionSequence && cursor.seq != null;
|
|
385
|
+
// Fetch progress is deliberately volatile. It moves through pages as soon as
|
|
386
|
+
// the server scans them (so an active job does not hot-poll one seen row),
|
|
387
|
+
// while `cursor` moves only after queue/recovery settlement. A restart always
|
|
388
|
+
// resumes from the durable cursor and therefore replays unfinished work.
|
|
389
|
+
const fetchCursor = { value: cursor.value, seq: cursor.seq };
|
|
390
|
+
// A capable server pages by monotonic mention ingestion sequence. Older
|
|
391
|
+
// servers remain timestamp batches and do not commit T until every accepted
|
|
392
|
+
// T sibling is settled.
|
|
393
|
+
const cursorBatches = new Map(); // mention-sequence/timestamp key → batch
|
|
394
|
+
const delegatedReplyIds = new Set();
|
|
395
|
+
|
|
396
|
+
const batchKey = (timestamp, mentionSeq) =>
|
|
397
|
+
mentionSequenceActive ? `seq:${mentionSeq}` : String(timestamp);
|
|
398
|
+
|
|
399
|
+
function registerCursorMention(message) {
|
|
400
|
+
const at = message?.created_at;
|
|
401
|
+
const timestamp = new Date(at).getTime();
|
|
402
|
+
if (!message?.id || !Number.isFinite(timestamp)) return;
|
|
403
|
+
const mentionSeq = Number.isSafeInteger(message.mentionSeq) && message.mentionSeq >= 0
|
|
404
|
+
? message.mentionSeq
|
|
405
|
+
: null;
|
|
406
|
+
// Deploy, dispatch, ambient, and DM projections are useful intake, but are
|
|
407
|
+
// not rows in the explicit mention scan. They must never advance that
|
|
408
|
+
// durable high-watermark.
|
|
409
|
+
if (serverMentionSequence && mentionSeq == null) return;
|
|
410
|
+
const key = batchKey(timestamp, mentionSeq);
|
|
411
|
+
let batch = cursorBatches.get(key);
|
|
412
|
+
if (!batch) {
|
|
413
|
+
batch = { at, timestamp, seq: mentionSeq, ids: new Set(), settled: new Set() };
|
|
414
|
+
cursorBatches.set(key, batch);
|
|
415
|
+
}
|
|
416
|
+
batch.ids.add(message.id);
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
function commitSettledCursorBatches() {
|
|
420
|
+
let cursorTime = cursor.value ? new Date(cursor.value).getTime() : -Infinity;
|
|
421
|
+
let advanced = false;
|
|
422
|
+
const ordered = [...cursorBatches.entries()].sort((left, right) => {
|
|
423
|
+
if (mentionSequenceActive) return left[1].seq - right[1].seq;
|
|
424
|
+
const time = left[1].timestamp - right[1].timestamp;
|
|
425
|
+
return time;
|
|
426
|
+
});
|
|
427
|
+
for (const [key, batch] of ordered) {
|
|
428
|
+
const behind = mentionSequenceActive
|
|
429
|
+
? cursor.seq != null && batch.seq <= cursor.seq
|
|
430
|
+
: batch.timestamp <= cursorTime;
|
|
431
|
+
if (behind) {
|
|
432
|
+
cursorBatches.delete(key);
|
|
433
|
+
continue;
|
|
434
|
+
}
|
|
435
|
+
if (batch.settled.size < batch.ids.size) break;
|
|
436
|
+
cursor.value = batch.at;
|
|
437
|
+
cursor.seq = batch.seq;
|
|
438
|
+
cursorTime = batch.timestamp;
|
|
439
|
+
cursorBatches.delete(key);
|
|
440
|
+
advanced = true;
|
|
441
|
+
if (mentionSequenceActive && (fetchCursor.seq == null || cursor.seq > fetchCursor.seq)) {
|
|
442
|
+
fetchCursor.value = cursor.value;
|
|
443
|
+
fetchCursor.seq = cursor.seq;
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
if (advanced && liveCfg.catchupMs > 0) {
|
|
447
|
+
saveMentionCursor(me.agentId, cursor.value, {
|
|
448
|
+
...(mentionCursorDir ? { dir: mentionCursorDir } : {}),
|
|
449
|
+
...(mentionSequenceActive ? { mentionSeq: cursor.seq } : {}),
|
|
450
|
+
log,
|
|
451
|
+
});
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
function settleCursorMention(message) {
|
|
456
|
+
const timestamp = new Date(message?.created_at).getTime();
|
|
457
|
+
const mentionSeq = Number.isSafeInteger(message?.mentionSeq) ? message.mentionSeq : null;
|
|
458
|
+
const batch = cursorBatches.get(batchKey(timestamp, mentionSeq));
|
|
459
|
+
if (!batch || !message?.id) return;
|
|
460
|
+
batch.settled.add(message.id);
|
|
461
|
+
commitSettledCursorBatches();
|
|
462
|
+
}
|
|
241
463
|
// 0860 — a DM message inside its settle window: the server withheld it and
|
|
242
464
|
// said when it matures. One shortened sleep picks it up right then, instead
|
|
243
465
|
// of the person waiting out the full poll interval mid-conversation.
|
|
244
466
|
let pollAgainMs = null;
|
|
467
|
+
let oldMentionReplayOnly = false;
|
|
245
468
|
|
|
246
469
|
// Live config: re-read between polls so model/permission/codingCmd edits to
|
|
247
470
|
// hilos-agent.json take effect without a restart. Identity stays pinned.
|
|
@@ -250,32 +473,52 @@ export async function run(cfg, { handler = handleTask, log = console, signal, on
|
|
|
250
473
|
// One-at-a-time worker so the poll loop NEVER blocks on a multi-minute run.
|
|
251
474
|
// The loop only fetches + enqueues; the queue runs jobs in order. This is what
|
|
252
475
|
// lets a "stop" (and new mentions) be picked up while a task is still running.
|
|
253
|
-
|
|
476
|
+
queue = createQueue({
|
|
254
477
|
concurrency: cfg.queueConcurrency || 1,
|
|
255
478
|
runJob: (job, { signal: jobSignal }) =>
|
|
256
479
|
job.kind === "reply-bridge"
|
|
257
480
|
? safeBridgeHandle(job.bridge, jobSignal)
|
|
258
481
|
: safeHandle(job.message, job.channelId, jobSignal),
|
|
482
|
+
// A handler can finish after the server committed its Iterate claim but
|
|
483
|
+
// before the local receipt was cleared. Never advance the FIFO until that
|
|
484
|
+
// exact durable owner has been reconciled.
|
|
485
|
+
afterJob: async (job) => {
|
|
486
|
+
const recoveryCheck = reconcileClaimsOnce("between queued jobs");
|
|
487
|
+
const safe = cfg.once
|
|
488
|
+
? await withinShutdownReconcileBound(recoveryCheck)
|
|
489
|
+
: await recoveryCheck;
|
|
490
|
+
if (safe && job.kind !== "reply-bridge" && job.message?.created_at) {
|
|
491
|
+
settleCursorMention(job.message);
|
|
492
|
+
}
|
|
493
|
+
if (!safe && cfg.once) {
|
|
494
|
+
// Cron/--once has no future poll cycle in which an infinite barrier can
|
|
495
|
+
// recover. Hand the durable receipt to the next invocation; unstarted
|
|
496
|
+
// queue entries remain unacknowledged and replay from the cursor.
|
|
497
|
+
void queue.shutdown("once-mode recovery handoff");
|
|
498
|
+
}
|
|
499
|
+
return safe;
|
|
500
|
+
},
|
|
501
|
+
afterJobRetryDelay: iterateClaimRecoveryRetryDelay,
|
|
259
502
|
});
|
|
260
503
|
|
|
261
504
|
let boundThreadRoots = new Set();
|
|
262
505
|
let replyScanOffset = 0;
|
|
263
506
|
|
|
264
|
-
// Stopping the embedded daemon: the moment the caller aborts
|
|
507
|
+
// Stopping the embedded daemon: the moment the caller aborts or the MCP client
|
|
508
|
+
// latches a terminal auth response, cancel the active
|
|
265
509
|
// job (via the queue's existing cancel path — it aborts the in-flight job's
|
|
266
510
|
// signal) so a multi-minute run is torn down promptly, and emit a stopping
|
|
267
511
|
// event. New tasks are refused in intake(); the loop below exits on the same
|
|
268
|
-
// signal
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
}
|
|
512
|
+
// signal. Caller cancellation resolves cleanly; terminal auth rejects after
|
|
513
|
+
// the same cleanup so the operator sees that the token must be replaced.
|
|
514
|
+
runSignal.addEventListener(
|
|
515
|
+
"abort",
|
|
516
|
+
() => {
|
|
517
|
+
emitStopping();
|
|
518
|
+
void queue.shutdown("run() aborted");
|
|
519
|
+
},
|
|
520
|
+
{ once: true },
|
|
521
|
+
);
|
|
279
522
|
|
|
280
523
|
async function safeHandle(message, channelId, jobSignal) {
|
|
281
524
|
emit({ type: "task-start", channelId, messageId: message.id, text: (message.body || "").slice(0, 200) });
|
|
@@ -315,7 +558,12 @@ export async function run(cfg, { handler = handleTask, log = console, signal, on
|
|
|
315
558
|
try {
|
|
316
559
|
log.log(`→ task in ${channelId}: "${(message.body || "").slice(0, 80)}"`);
|
|
317
560
|
// liveCfg so a job uses the latest model/permission/codingCmd at run time.
|
|
318
|
-
const result = await handler(
|
|
561
|
+
const result = await handler(
|
|
562
|
+
{ message: withImages, channelId, tool, me, caps, iterateClaimRecoveryStore },
|
|
563
|
+
liveCfg,
|
|
564
|
+
undefined,
|
|
565
|
+
{ signal: jobSignal, onStopRequested },
|
|
566
|
+
);
|
|
319
567
|
emit({ type: "task-done", channelId, status: result && result.status });
|
|
320
568
|
} catch (e) {
|
|
321
569
|
emit({ type: "task-error", channelId, error: e?.message || String(e) });
|
|
@@ -360,7 +608,7 @@ export async function run(cfg, { handler = handleTask, log = console, signal, on
|
|
|
360
608
|
// when it lands behind work already in progress.
|
|
361
609
|
async function intake(m, channelId) {
|
|
362
610
|
// Once we're stopping, don't start anything new.
|
|
363
|
-
if (
|
|
611
|
+
if (runSignal.aborted) return { accepted: false };
|
|
364
612
|
if (looksLikeCancel(m.body) && (queue.active() || queue.size())) {
|
|
365
613
|
queue.cancelActive("user asked to stop");
|
|
366
614
|
await tool("post_message", {
|
|
@@ -368,12 +616,18 @@ export async function run(cfg, { handler = handleTask, log = console, signal, on
|
|
|
368
616
|
parentId: m.parentId ?? null,
|
|
369
617
|
body: "Stopping the current task.",
|
|
370
618
|
}).catch(() => {});
|
|
371
|
-
return;
|
|
619
|
+
return { accepted: false, settled: true };
|
|
372
620
|
}
|
|
373
|
-
const
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
621
|
+
const mentionSeq = Number.isSafeInteger(m?.mentionSeq) && m.mentionSeq >= 0
|
|
622
|
+
? m.mentionSeq
|
|
623
|
+
: null;
|
|
624
|
+
const queueKey = mentionSequenceActive && mentionSeq != null
|
|
625
|
+
? `${dedupeKey(m, channelId)}::mention-seq:${mentionSeq}`
|
|
626
|
+
: dedupeKey(m, channelId);
|
|
627
|
+
const r = queue.enqueue({ message: m, channelId, key: queueKey });
|
|
628
|
+
if (!r.accepted) {
|
|
629
|
+
if (r.deduped) log.log(" (deduped a repeat of an in-flight/queued ask)");
|
|
630
|
+
return { accepted: false, settled: Boolean(r.deduped) };
|
|
377
631
|
}
|
|
378
632
|
// A report-card deploy decision is already visible in place. list_mentions
|
|
379
633
|
// projects the pending decision into this structured event, so don't add a
|
|
@@ -388,6 +642,7 @@ export async function run(cfg, { handler = handleTask, log = console, signal, on
|
|
|
388
642
|
body: "Got it — queued behind the current task.",
|
|
389
643
|
}).catch(() => {});
|
|
390
644
|
}
|
|
645
|
+
return { accepted: true, settled: false };
|
|
391
646
|
}
|
|
392
647
|
|
|
393
648
|
// Long-poll (0866): ask a capable server to HOLD the request and answer the
|
|
@@ -396,50 +651,123 @@ export async function run(cfg, { handler = handleTask, log = console, signal, on
|
|
|
396
651
|
const mentionWaitMs = () =>
|
|
397
652
|
serverMentionWait && liveCfg.longPollMs > 0 ? Math.floor(liveCfg.longPollMs) : 0;
|
|
398
653
|
|
|
399
|
-
async function passViaMentions() {
|
|
654
|
+
async function passViaMentions(immediatePage = 1) {
|
|
400
655
|
const waitMs = mentionWaitMs();
|
|
401
656
|
const mentionArgs = {
|
|
402
|
-
...(
|
|
657
|
+
...(fetchCursor.value ? { since: fetchCursor.value } : {}),
|
|
658
|
+
...(mentionSequenceActive && fetchCursor.seq != null
|
|
659
|
+
? { sinceMentionSeq: fetchCursor.seq }
|
|
660
|
+
: {}),
|
|
661
|
+
...(serverMentionSequenceBootstrap && !mentionSequenceActive
|
|
662
|
+
? { initializeMentionSeq: true }
|
|
663
|
+
: {}),
|
|
403
664
|
...(cfg.channelId ? { channelId: cfg.channelId } : {}),
|
|
404
665
|
...(waitMs ? { waitMs } : {}),
|
|
405
666
|
};
|
|
406
|
-
const out = await tool("list_mentions", mentionArgs);
|
|
667
|
+
const out = await tool("list_mentions", mentionArgs, { signal: runSignal });
|
|
668
|
+
const mentions = (out?.mentions ?? []).slice().reverse();
|
|
669
|
+
const announcedSequence = out?.mentionCursor?.sequence;
|
|
670
|
+
const explicitMentions = mentions.filter(
|
|
671
|
+
(message) => Number.isSafeInteger(message?.mentionSeq) && message.mentionSeq >= 0,
|
|
672
|
+
);
|
|
673
|
+
let bootstrappedSequence = false;
|
|
674
|
+
let settledEmptySequencePage = false;
|
|
675
|
+
if (
|
|
676
|
+
mentionSequenceActive &&
|
|
677
|
+
Number.isSafeInteger(announcedSequence) &&
|
|
678
|
+
announcedSequence >= 0 &&
|
|
679
|
+
(fetchCursor.seq == null || announcedSequence > fetchCursor.seq)
|
|
680
|
+
) {
|
|
681
|
+
fetchCursor.seq = announcedSequence;
|
|
682
|
+
// The server scanned explicit rows through this sequence but none
|
|
683
|
+
// survived its authoritative filters/revalidation. With no older batch
|
|
684
|
+
// still unsettled, advancing is a durable settlement rather than merely
|
|
685
|
+
// volatile fetch progress; otherwise --once would reload the same empty
|
|
686
|
+
// page forever and starve valid work behind it.
|
|
687
|
+
if (
|
|
688
|
+
explicitMentions.length === 0 &&
|
|
689
|
+
cursorBatches.size === 0 &&
|
|
690
|
+
(cursor.seq == null || announcedSequence > cursor.seq)
|
|
691
|
+
) {
|
|
692
|
+
cursor.seq = announcedSequence;
|
|
693
|
+
if (!cursor.value) cursor.value = new Date().toISOString();
|
|
694
|
+
fetchCursor.value = cursor.value;
|
|
695
|
+
if (liveCfg.catchupMs > 0) {
|
|
696
|
+
saveMentionCursor(me.agentId, cursor.value, {
|
|
697
|
+
...(mentionCursorDir ? { dir: mentionCursorDir } : {}),
|
|
698
|
+
mentionSeq: cursor.seq,
|
|
699
|
+
log,
|
|
700
|
+
});
|
|
701
|
+
}
|
|
702
|
+
settledEmptySequencePage = true;
|
|
703
|
+
}
|
|
704
|
+
} else if (
|
|
705
|
+
serverMentionSequenceBootstrap &&
|
|
706
|
+
!mentionSequenceActive &&
|
|
707
|
+
Number.isSafeInteger(announcedSequence) &&
|
|
708
|
+
announcedSequence >= 0 &&
|
|
709
|
+
explicitMentions.length === 0 &&
|
|
710
|
+
cursorBatches.size === 0
|
|
711
|
+
) {
|
|
712
|
+
mentionSequenceActive = true;
|
|
713
|
+
cursor.seq = announcedSequence;
|
|
714
|
+
fetchCursor.seq = announcedSequence;
|
|
715
|
+
if (!cursor.value) cursor.value = new Date().toISOString();
|
|
716
|
+
fetchCursor.value = cursor.value;
|
|
717
|
+
if (liveCfg.catchupMs > 0) {
|
|
718
|
+
saveMentionCursor(me.agentId, cursor.value, {
|
|
719
|
+
...(mentionCursorDir ? { dir: mentionCursorDir } : {}),
|
|
720
|
+
mentionSeq: cursor.seq,
|
|
721
|
+
log,
|
|
722
|
+
});
|
|
723
|
+
}
|
|
724
|
+
bootstrappedSequence = true;
|
|
725
|
+
}
|
|
407
726
|
// Floor the server's settle hint so a clock skew can never hot-loop the poll.
|
|
408
727
|
pollAgainMs =
|
|
409
728
|
typeof out?.retryAfterMs === "number" && out.retryAfterMs > 0
|
|
410
729
|
? Math.max(500, out.retryAfterMs)
|
|
411
730
|
: null;
|
|
412
|
-
const
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
731
|
+
const mentionSeenKey = (message) =>
|
|
732
|
+
mentionSequenceActive && Number.isSafeInteger(message?.mentionSeq) && message.mentionSeq >= 0
|
|
733
|
+
? `${message.id}:mention-seq:${message.mentionSeq}`
|
|
734
|
+
: message.id;
|
|
735
|
+
const fresh = mentions.filter((message) => !seen.has(mentionSeenKey(message)));
|
|
736
|
+
oldMentionReplayOnly = !mentionSequenceActive && mentions.length > 0 && fresh.length === 0;
|
|
737
|
+
// Register the WHOLE response before enqueueing its first item. A fast
|
|
738
|
+
// first handler can otherwise settle timestamp T before its T sibling is
|
|
739
|
+
// even known locally, advancing a strict timestamp cursor too early.
|
|
740
|
+
for (const message of fresh) registerCursorMention(message);
|
|
741
|
+
for (const m of fresh) {
|
|
742
|
+
const seenKey = mentionSeenKey(m);
|
|
743
|
+
seen.add(seenKey);
|
|
417
744
|
// A human reply in a locally-bound thread belongs to the provider
|
|
418
745
|
// session that created that anchor — even when the person also typed an
|
|
419
746
|
// @mention. Let the bridge take it; starting a fresh repo task here would
|
|
420
747
|
// duplicate work and lose the session context this feature exists for.
|
|
421
748
|
if (m.parentId && boundThreadRoots.has(m.parentId)) {
|
|
422
|
-
if (
|
|
423
|
-
|
|
424
|
-
|
|
749
|
+
if (delegatedReplyIds.has(m.id)) {
|
|
750
|
+
settleCursorMention(m);
|
|
751
|
+
} else {
|
|
752
|
+
// Delegation did not become durable in this pass. Let the next poll
|
|
753
|
+
// retry it instead of pinning an un-settleable seen entry forever.
|
|
754
|
+
seen.delete(seenKey);
|
|
425
755
|
}
|
|
426
756
|
continue;
|
|
427
757
|
}
|
|
428
|
-
await intake(m, m.channelId);
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
if (!cursor.value || new Date(m.created_at).getTime() > new Date(cursor.value).getTime()) {
|
|
432
|
-
cursor.value = m.created_at;
|
|
433
|
-
advanced = true;
|
|
434
|
-
}
|
|
758
|
+
const accepted = await intake(m, m.channelId);
|
|
759
|
+
if (accepted?.settled) settleCursorMention(m);
|
|
760
|
+
if (!accepted?.accepted && !accepted?.settled) seen.delete(seenKey);
|
|
435
761
|
}
|
|
436
|
-
//
|
|
437
|
-
//
|
|
438
|
-
//
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
762
|
+
// The initialization response intentionally contains no explicit mention.
|
|
763
|
+
// Follow it with one sequence pass immediately so `--once` still performs
|
|
764
|
+
// useful work and a normal daemon does not spend a poll interval idle.
|
|
765
|
+
if (
|
|
766
|
+
(bootstrappedSequence || settledEmptySequencePage) &&
|
|
767
|
+
!runSignal.aborted &&
|
|
768
|
+
immediatePage < MAX_IMMEDIATE_SEQUENCE_PAGES
|
|
769
|
+
) {
|
|
770
|
+
await passViaMentions(immediatePage + 1);
|
|
443
771
|
}
|
|
444
772
|
}
|
|
445
773
|
|
|
@@ -460,7 +788,8 @@ export async function run(cfg, { handler = handleTask, log = console, signal, on
|
|
|
460
788
|
const ids = bridge.replies.map((reply) => reply.id).filter(Boolean);
|
|
461
789
|
const key = `reply-bridge:${bridge.binding.threadRootId}:${ids.join(",")}`;
|
|
462
790
|
const queued = queue.enqueue({ kind: "reply-bridge", bridge, key });
|
|
463
|
-
if (queued.deduped) continue;
|
|
791
|
+
if (!queued.accepted && !queued.deduped) continue;
|
|
792
|
+
for (const id of ids) delegatedReplyIds.add(id);
|
|
464
793
|
if (liveCfg.queueAcks && !queued.startedImmediately) {
|
|
465
794
|
await tool("post_message", {
|
|
466
795
|
channelId: bridge.binding.channelId,
|
|
@@ -485,7 +814,7 @@ export async function run(cfg, { handler = handleTask, log = console, signal, on
|
|
|
485
814
|
}
|
|
486
815
|
|
|
487
816
|
do {
|
|
488
|
-
if (
|
|
817
|
+
if (runSignal.aborted) break;
|
|
489
818
|
// Pick up live edits to hilos-agent.json (model/permission/codingCmd/etc.)
|
|
490
819
|
// before each pass. Guarded: a bad file leaves liveCfg untouched.
|
|
491
820
|
try {
|
|
@@ -500,10 +829,33 @@ export async function run(cfg, { handler = handleTask, log = console, signal, on
|
|
|
500
829
|
}
|
|
501
830
|
let passFailed = false;
|
|
502
831
|
try {
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
832
|
+
// A live queue job may have persisted its claim journal immediately
|
|
833
|
+
// before sending claim_local_run_iterate. Releasing that record from the
|
|
834
|
+
// poll loop would race the job that still owns it. Keep polling while a
|
|
835
|
+
// job runs (Stop must remain responsive), and reconcile on the first
|
|
836
|
+
// idle pass instead. A queued-but-not-active job cannot have written a
|
|
837
|
+
// claim record yet, so active() is the precise boundary.
|
|
838
|
+
const recovery = !iterateClaimRecoveryStore
|
|
839
|
+
? { unresolved: 0, recoveryUnavailable: true }
|
|
840
|
+
: queue.active()
|
|
841
|
+
? { unresolved: 0, deferredWhileActive: true }
|
|
842
|
+
: await reconcileDaemonIterateClaims({
|
|
843
|
+
tool,
|
|
844
|
+
recoveryStore: iterateClaimRecoveryStore,
|
|
845
|
+
log,
|
|
846
|
+
});
|
|
847
|
+
if (!recovery.deferredWhileActive && recovery.unresolved > 0) {
|
|
848
|
+
passFailed = true;
|
|
849
|
+
log.error(
|
|
850
|
+
`iterate claim recovery unresolved (${recovery.unresolved}); mention intake remains paused`,
|
|
851
|
+
);
|
|
852
|
+
} else {
|
|
853
|
+
await passViaReplyBridge();
|
|
854
|
+
if (useMentions) await passViaMentions();
|
|
855
|
+
else await passViaScan();
|
|
856
|
+
}
|
|
506
857
|
} catch (e) {
|
|
858
|
+
if (isAuthStop(e)) throw e;
|
|
507
859
|
passFailed = true;
|
|
508
860
|
log.error(`poll error: ${e.message}`);
|
|
509
861
|
}
|
|
@@ -512,7 +864,7 @@ export async function run(cfg, { handler = handleTask, log = console, signal, on
|
|
|
512
864
|
await queue.idle();
|
|
513
865
|
break;
|
|
514
866
|
}
|
|
515
|
-
if (
|
|
867
|
+
if (runSignal.aborted) break;
|
|
516
868
|
// Interruptible so an abort during the sleep returns promptly instead of
|
|
517
869
|
// waiting out the full poll interval, and the doorbell (0824) still cuts
|
|
518
870
|
// any sleep short. A long-polled pass (0866) already spent its waiting
|
|
@@ -520,16 +872,50 @@ export async function run(cfg, { handler = handleTask, log = console, signal, on
|
|
|
520
872
|
// the pass FAILED, where re-holding immediately would hammer a hiccuping
|
|
521
873
|
// server; an error always backs off by the full pollMs. A settling DM
|
|
522
874
|
// (0860) shortens one sleep to the server's hint in either mode.
|
|
523
|
-
const held = useMentions && mentionWaitMs() > 0 && !passFailed;
|
|
875
|
+
const held = useMentions && mentionWaitMs() > 0 && !passFailed && !oldMentionReplayOnly;
|
|
524
876
|
const base = held ? 250 : liveCfg.pollMs;
|
|
525
877
|
await interruptibleSleep(pollAgainMs != null ? Math.min(base, pollAgainMs) : base);
|
|
526
|
-
} while (!
|
|
878
|
+
} while (!runSignal.aborted);
|
|
527
879
|
|
|
528
|
-
//
|
|
529
|
-
//
|
|
530
|
-
//
|
|
531
|
-
|
|
532
|
-
if (
|
|
533
|
-
|
|
880
|
+
// A terminal response may first surface inside a concurrently running
|
|
881
|
+
// handler while the poll loop is sleeping. The client latch aborts the run
|
|
882
|
+
// immediately; preserve the original typed error after the loop has fenced
|
|
883
|
+
// intake instead of turning that shutdown into a successful exit.
|
|
884
|
+
if (isAuthStop(stopSignal.reason)) throw stopSignal.reason;
|
|
885
|
+
|
|
886
|
+
// Clean stop: close the doorbell socket. The finally barrier below fences
|
|
887
|
+
// intake, discards pending work, and awaits active teardown before releasing
|
|
888
|
+
// this daemon's recovery-scope lock.
|
|
889
|
+
} catch (error) {
|
|
890
|
+
if (reportAuthStop(error)) throw error;
|
|
891
|
+
if (!canceledByRunSignal(error)) throw error;
|
|
892
|
+
} finally {
|
|
893
|
+
detachLifecycleSources();
|
|
894
|
+
wake?.stop();
|
|
895
|
+
await queue?.shutdown("run() shutting down");
|
|
896
|
+
let shutdownReconciled = !iterateClaimRecoveryStore || isAuthStop(stopSignal.reason);
|
|
897
|
+
const shutdownAttempts = Math.max(
|
|
898
|
+
1,
|
|
899
|
+
Math.floor(iterateClaimShutdownReconcileAttempts),
|
|
900
|
+
);
|
|
901
|
+
for (
|
|
902
|
+
let attempt = 0;
|
|
903
|
+
!shutdownReconciled && !isAuthStop(stopSignal.reason) && attempt < shutdownAttempts;
|
|
904
|
+
attempt += 1
|
|
905
|
+
) {
|
|
906
|
+
shutdownReconciled = await withinShutdownReconcileBound(
|
|
907
|
+
reconcileClaimsOnce("before daemon lock release"),
|
|
908
|
+
);
|
|
909
|
+
if (!shutdownReconciled && attempt + 1 < shutdownAttempts) {
|
|
910
|
+
await withinShutdownReconcileBound(iterateClaimRecoveryRetryDelay());
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
if (!shutdownReconciled && iterateClaimRecoveryStore) {
|
|
914
|
+
log.error(
|
|
915
|
+
"iterate claim recovery remains durable after bounded shutdown reconciliation; the next daemon will retry it",
|
|
916
|
+
);
|
|
917
|
+
}
|
|
918
|
+
iterateClaimRecoveryStore?.releaseLock();
|
|
919
|
+
if (isAuthStop(stopSignal.reason)) throw stopSignal.reason;
|
|
534
920
|
}
|
|
535
921
|
}
|