@dorokuma/herdsman 0.11.6 → 0.13.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -13
- package/dist/src/agent-history/pi-reader.d.ts.map +1 -1
- package/dist/src/agent-history/pi-reader.js +10 -2
- package/dist/src/agent-history/pi-reader.js.map +1 -1
- package/dist/src/agent-history/readers.js +6 -1
- package/dist/src/agent-history/readers.js.map +1 -1
- package/dist/src/agent-history/text.d.ts.map +1 -1
- package/dist/src/agent-history/text.js +3 -0
- package/dist/src/agent-history/text.js.map +1 -1
- package/dist/src/cli/herdsman.d.ts +14 -1
- package/dist/src/cli/herdsman.d.ts.map +1 -1
- package/dist/src/cli/herdsman.js +49 -65
- package/dist/src/cli/herdsman.js.map +1 -1
- package/dist/src/config/schema.d.ts +4 -0
- package/dist/src/config/schema.d.ts.map +1 -1
- package/dist/src/config/schema.js +5 -0
- package/dist/src/config/schema.js.map +1 -1
- package/dist/src/daemon/agent-event-reconciler.d.ts.map +1 -1
- package/dist/src/daemon/agent-event-reconciler.js +53 -2
- package/dist/src/daemon/agent-event-reconciler.js.map +1 -1
- package/dist/src/daemon/herdr-session-watch-manager.d.ts +13 -1
- package/dist/src/daemon/herdr-session-watch-manager.d.ts.map +1 -1
- package/dist/src/daemon/herdr-session-watch-manager.js +52 -15
- package/dist/src/daemon/herdr-session-watch-manager.js.map +1 -1
- package/dist/src/daemon/observability-server.d.ts.map +1 -1
- package/dist/src/daemon/observability-server.js +17 -1
- package/dist/src/daemon/observability-server.js.map +1 -1
- package/dist/src/daemon/process-manager.d.ts +50 -49
- package/dist/src/daemon/process-manager.d.ts.map +1 -1
- package/dist/src/daemon/process-manager.js +246 -204
- package/dist/src/daemon/process-manager.js.map +1 -1
- package/dist/src/daemon/service-supervision.d.ts +68 -0
- package/dist/src/daemon/service-supervision.d.ts.map +1 -0
- package/dist/src/daemon/service-supervision.js +127 -0
- package/dist/src/daemon/service-supervision.js.map +1 -0
- package/dist/src/daemon/service.d.ts +35 -0
- package/dist/src/daemon/service.d.ts.map +1 -1
- package/dist/src/daemon/service.js +185 -27
- package/dist/src/daemon/service.js.map +1 -1
- package/dist/src/db/agent-events.d.ts +31 -0
- package/dist/src/db/agent-events.d.ts.map +1 -1
- package/dist/src/db/agent-events.js +207 -55
- package/dist/src/db/agent-events.js.map +1 -1
- package/dist/src/db/agents.js +1 -1
- package/dist/src/db/agents.js.map +1 -1
- package/dist/src/db/status-event-plans.d.ts +36 -4
- package/dist/src/db/status-event-plans.d.ts.map +1 -1
- package/dist/src/db/status-event-plans.js +96 -11
- package/dist/src/db/status-event-plans.js.map +1 -1
- package/dist/src/herdr/session-list.d.ts +11 -0
- package/dist/src/herdr/session-list.d.ts.map +1 -1
- package/dist/src/herdr/session-list.js +13 -0
- package/dist/src/herdr/session-list.js.map +1 -1
- package/dist/src/herdr/socket-client.d.ts +8 -0
- package/dist/src/herdr/socket-client.d.ts.map +1 -1
- package/dist/src/herdr/socket-client.js +43 -0
- package/dist/src/herdr/socket-client.js.map +1 -1
- package/dist/src/observability/agent-index-service.d.ts +16 -0
- package/dist/src/observability/agent-index-service.d.ts.map +1 -1
- package/dist/src/observability/agent-index-service.js +837 -114
- package/dist/src/observability/agent-index-service.js.map +1 -1
- package/dist/src/observability/agent-orchestrator-service.d.ts.map +1 -1
- package/dist/src/observability/agent-orchestrator-service.js +5 -1
- package/dist/src/observability/agent-orchestrator-service.js.map +1 -1
- package/dist/src/observability/contracts.d.ts +6 -1
- package/dist/src/observability/contracts.d.ts.map +1 -1
- package/dist/src/observability/contracts.js +8 -0
- package/dist/src/observability/contracts.js.map +1 -1
- package/dist/src/observability/schemas.d.ts +1 -0
- package/dist/src/observability/schemas.d.ts.map +1 -1
- package/dist/src/observability/schemas.js +1 -0
- package/dist/src/observability/schemas.js.map +1 -1
- package/dist/src/observability/turn-completion.d.ts +3 -0
- package/dist/src/observability/turn-completion.d.ts.map +1 -1
- package/dist/src/observability/turn-completion.js +12 -2
- package/dist/src/observability/turn-completion.js.map +1 -1
- package/drizzle/0010_legacy_degraded_cleanup.sql +1 -0
- package/drizzle/meta/0010_snapshot.json +975 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +3 -6
- package/README.ja.md +0 -145
|
@@ -6,7 +6,7 @@ import { runSqliteTransaction } from "../db/client.js";
|
|
|
6
6
|
import { normalizeHerdrSessionSnapshot } from "../herdr/session-snapshot.js";
|
|
7
7
|
import { HerdrSocketClient } from "../herdr/socket-client.js";
|
|
8
8
|
import { AgentContextService } from "../observability/agent-context-service.js";
|
|
9
|
-
import { parseAgentStatus, } from "../observability/contracts.js";
|
|
9
|
+
import { parseAgentStatus, VALID_AGENT_EVENT_TYPES, } from "../observability/contracts.js";
|
|
10
10
|
import { TURN_SIGNAL_WAIT_MS } from "../observability/turn-completion.js";
|
|
11
11
|
export function historyHasAdvanced(current, baseline, options) {
|
|
12
12
|
if (current.lastAssistantMessage?.ref &&
|
|
@@ -22,6 +22,10 @@ export function historyHasAdvanced(current, baseline, options) {
|
|
|
22
22
|
}
|
|
23
23
|
return current.messageCount > (baseline?.messageCount ?? 0);
|
|
24
24
|
}
|
|
25
|
+
function isTerminalAssistant(compact) {
|
|
26
|
+
const stopReason = compact?.lastAssistantMessage?.stopReason;
|
|
27
|
+
return stopReason === "stop" || stopReason === "length";
|
|
28
|
+
}
|
|
25
29
|
export const PLAN_WAITING_HISTORY = "PLAN_WAITING_HISTORY";
|
|
26
30
|
export class PlanWaitingHistoryError extends Error {
|
|
27
31
|
constructor() {
|
|
@@ -29,6 +33,17 @@ export class PlanWaitingHistoryError extends Error {
|
|
|
29
33
|
this.name = "PlanWaitingHistoryError";
|
|
30
34
|
}
|
|
31
35
|
}
|
|
36
|
+
/**
|
|
37
|
+
* Invariant for the waiting-history retry ring: only a genuine
|
|
38
|
+
* `PlanWaitingHistoryError` may (re)arm the 10s timer. It is the single signal
|
|
39
|
+
* that the agent history legitimately has not advanced yet. Synthetic retry
|
|
40
|
+
* reasons such as `degraded` are still recorded on the row, but they never
|
|
41
|
+
* authorize another spin: the pending row is drained by the next refresh
|
|
42
|
+
* instead, so a degraded round cannot keep the ring alive on its own.
|
|
43
|
+
*/
|
|
44
|
+
function retryRingAuthorized(error) {
|
|
45
|
+
return error instanceof PlanWaitingHistoryError;
|
|
46
|
+
}
|
|
32
47
|
/**
|
|
33
48
|
* Sentinel returned by #appendStatusEvents when the plan must be CANCELLED
|
|
34
49
|
* rather than skipped (skip still maps to completed in #runPlanRow). The agent
|
|
@@ -47,16 +62,23 @@ export class AgentIndexService {
|
|
|
47
62
|
#refreshInFlightBySession = new Map();
|
|
48
63
|
#scheduleRetry;
|
|
49
64
|
#sessionOperationTail = new Map();
|
|
65
|
+
#shutdownSignal;
|
|
50
66
|
#sleep;
|
|
51
67
|
#turnCompletions;
|
|
52
68
|
#waitingHistoryTimers = new Map();
|
|
69
|
+
#now;
|
|
70
|
+
#onAgentEvent;
|
|
53
71
|
constructor(options) {
|
|
54
72
|
this.#clearRetry = options.clearRetry ?? ((timer) => clearTimeout(timer));
|
|
55
73
|
this.#clientFactory = options.clientFactory ?? ((input) => new HerdrSocketClient(input));
|
|
56
74
|
this.#scheduleRetry = options.scheduleRetry ?? ((cb, delay) => setTimeout(cb, delay));
|
|
57
75
|
this.#sleep = options.sleep ?? sleep;
|
|
76
|
+
this.#shutdownSignal = options.shutdownSignal;
|
|
77
|
+
this.#shutdownSignal?.addEventListener("abort", () => this.abortPendingWaits(), { once: true });
|
|
78
|
+
this.#now = options.now ?? (() => Date.now());
|
|
58
79
|
this.#stores = options.stores;
|
|
59
80
|
this.#turnCompletions = options.turnCompletions;
|
|
81
|
+
this.#onAgentEvent = options.onAgentEvent ?? (() => { });
|
|
60
82
|
if (options.context) {
|
|
61
83
|
this.#context = options.context;
|
|
62
84
|
}
|
|
@@ -127,6 +149,21 @@ export class AgentIndexService {
|
|
|
127
149
|
}
|
|
128
150
|
this.#waitingHistoryTimers.clear();
|
|
129
151
|
}
|
|
152
|
+
/**
|
|
153
|
+
* Aborts every in-flight status wait (history advance, turn completion,
|
|
154
|
+
* readiness polling). Called on shutdown through the shutdown signal; the
|
|
155
|
+
* waiters treat an abort exactly like a closed pane, so no event is appended
|
|
156
|
+
* for a half-observed transition and the plan row stays retryable.
|
|
157
|
+
*/
|
|
158
|
+
abortPendingWaits() {
|
|
159
|
+
for (const [key, controllers] of this.#activeWaiters) {
|
|
160
|
+
for (const controller of controllers) {
|
|
161
|
+
controller.abort();
|
|
162
|
+
}
|
|
163
|
+
controllers.clear();
|
|
164
|
+
this.#activeWaiters.delete(key);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
130
167
|
async drainInFlightPlans() {
|
|
131
168
|
const tails = Array.from(this.#planTailByAgent.values());
|
|
132
169
|
await Promise.allSettled(tails);
|
|
@@ -146,14 +183,36 @@ export class AgentIndexService {
|
|
|
146
183
|
if (!store)
|
|
147
184
|
return;
|
|
148
185
|
const current = store.get(planId);
|
|
149
|
-
if (current.status !== "pending"
|
|
150
|
-
this.#
|
|
186
|
+
if (current.status !== "pending") {
|
|
187
|
+
this.#clearWaitingTimer(planId);
|
|
151
188
|
return;
|
|
152
189
|
}
|
|
153
190
|
await this.#enqueueAgentPlan(plan.agent.id, () => this.#retryWaitingPlanRow(planId, plan));
|
|
154
191
|
}
|
|
155
192
|
catch (error) {
|
|
156
|
-
|
|
193
|
+
// A failed retry callback used to be a silent `console.debug`; the row it
|
|
194
|
+
// was supposed to advance can then sit in `pending` forever with no
|
|
195
|
+
// signal at all, so the failure is reported as a warning with the row
|
|
196
|
+
// identity attached.
|
|
197
|
+
console.warn("Herdsman waiting history retry callback failed", {
|
|
198
|
+
planId,
|
|
199
|
+
agentId: plan.agent.id,
|
|
200
|
+
herdrSessionName: plan.agent.herdrSessionName,
|
|
201
|
+
paneId: plan.agent.paneId,
|
|
202
|
+
error,
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
finally {
|
|
206
|
+
const store = this.#stores.statusEventPlans;
|
|
207
|
+
if (store) {
|
|
208
|
+
const current = store.get(planId);
|
|
209
|
+
if (current.status === "completed" ||
|
|
210
|
+
current.status === "cancelled" ||
|
|
211
|
+
current.status === "failed" ||
|
|
212
|
+
current.status === "discarded") {
|
|
213
|
+
this.#clearWaitingTimer(planId);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
157
216
|
}
|
|
158
217
|
}, 10_000);
|
|
159
218
|
this.#waitingHistoryTimers.set(planId, timer);
|
|
@@ -167,11 +226,7 @@ export class AgentIndexService {
|
|
|
167
226
|
current = store.get(planId);
|
|
168
227
|
}
|
|
169
228
|
catch {
|
|
170
|
-
this.#
|
|
171
|
-
return;
|
|
172
|
-
}
|
|
173
|
-
if (current.status !== "pending" || current.lastError !== PLAN_WAITING_HISTORY) {
|
|
174
|
-
this.#waitingHistoryTimers.delete(planId);
|
|
229
|
+
this.#clearWaitingTimer(planId);
|
|
175
230
|
return;
|
|
176
231
|
}
|
|
177
232
|
let retryPlan;
|
|
@@ -193,12 +248,44 @@ export class AgentIndexService {
|
|
|
193
248
|
agentId: plan.agent.id,
|
|
194
249
|
error: err,
|
|
195
250
|
});
|
|
196
|
-
store.
|
|
197
|
-
const
|
|
251
|
+
const currentPlan = store.get(planId);
|
|
252
|
+
const retryError = currentPlan.lastError === "degraded"
|
|
253
|
+
? new Error("degraded")
|
|
254
|
+
: new PlanWaitingHistoryError();
|
|
255
|
+
const updated = store.markRetry(planId, retryError);
|
|
256
|
+
if (!updated) {
|
|
257
|
+
this.#clearWaitingTimer(planId);
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
198
260
|
if (updated.status === "pending") {
|
|
199
|
-
|
|
261
|
+
// Invariant: a synthetic retry reason never re-arms the ring (see
|
|
262
|
+
// retryRingAuthorized); the pending row waits for the next refresh.
|
|
263
|
+
if (retryRingAuthorized(retryError))
|
|
264
|
+
this.#scheduleWaitingHistoryRetry(planId, plan);
|
|
265
|
+
}
|
|
266
|
+
else if (updated.status === "discarded") {
|
|
267
|
+
this.#clearWaitingTimer(planId);
|
|
268
|
+
console.info("Herdsman status event plan discarded after max attempts", {
|
|
269
|
+
planId,
|
|
270
|
+
agentId: plan.agent.id,
|
|
271
|
+
attempts: updated.attempts,
|
|
272
|
+
from: plan.from,
|
|
273
|
+
to: plan.to,
|
|
274
|
+
herdrSessionName: plan.agent.herdrSessionName,
|
|
275
|
+
});
|
|
276
|
+
this.#appendPlanDiscardedEvent({
|
|
277
|
+
agent: plan.agent,
|
|
278
|
+
attempts: updated.attempts,
|
|
279
|
+
compactHistory: plan.compactHistory,
|
|
280
|
+
from: plan.from,
|
|
281
|
+
planCreatedAt: currentPlan.createdAt.getTime(),
|
|
282
|
+
planId,
|
|
283
|
+
reason: updated.lastError,
|
|
284
|
+
to: plan.to,
|
|
285
|
+
});
|
|
200
286
|
}
|
|
201
287
|
else if (updated.status === "failed") {
|
|
288
|
+
this.#clearWaitingTimer(planId);
|
|
202
289
|
console.warn("Herdsman status event plan failed after max attempts", {
|
|
203
290
|
planId,
|
|
204
291
|
agentId: plan.agent.id,
|
|
@@ -207,19 +294,26 @@ export class AgentIndexService {
|
|
|
207
294
|
to: plan.to,
|
|
208
295
|
herdrSessionName: plan.agent.herdrSessionName,
|
|
209
296
|
});
|
|
210
|
-
this.#appendPlanFailedEvent({
|
|
297
|
+
const failedEvent = this.#appendPlanFailedEvent({
|
|
211
298
|
agent: plan.agent,
|
|
212
299
|
attempts: updated.attempts,
|
|
213
300
|
compactHistory: plan.compactHistory,
|
|
214
301
|
from: plan.from,
|
|
302
|
+
planCreatedAt: currentPlan.createdAt.getTime(),
|
|
215
303
|
planId,
|
|
216
304
|
reason: updated.lastError,
|
|
217
305
|
to: plan.to,
|
|
218
306
|
});
|
|
307
|
+
if (VALID_AGENT_EVENT_TYPES.has(failedEvent.type)) {
|
|
308
|
+
this.#onAgentEvent?.(failedEvent);
|
|
309
|
+
}
|
|
219
310
|
}
|
|
220
311
|
return;
|
|
221
312
|
}
|
|
222
|
-
await this.#runPlanRow(current, retryPlan);
|
|
313
|
+
const event = await this.#runPlanRow(current, retryPlan);
|
|
314
|
+
if (event && VALID_AGENT_EVENT_TYPES.has(event.type)) {
|
|
315
|
+
this.#onAgentEvent?.(event);
|
|
316
|
+
}
|
|
223
317
|
}
|
|
224
318
|
async executeStatusEventPlan(plan) {
|
|
225
319
|
if (plan.from === plan.to)
|
|
@@ -247,7 +341,14 @@ export class AgentIndexService {
|
|
|
247
341
|
to: plan.to,
|
|
248
342
|
...(plan.compactHistory ? { compactHistory: plan.compactHistory } : {}),
|
|
249
343
|
});
|
|
250
|
-
|
|
344
|
+
const result = await this.#enqueueAgentPlan(plan.agent.id, () => this.#runPlanRow(inserted, plan));
|
|
345
|
+
// Publication stays with the caller: every caller forwards this return value
|
|
346
|
+
// (`HerdrSessionWatchManager#submitPlan` -> `publishAgentEvent`, or the
|
|
347
|
+
// non-fast refresh/handle wrappers putting it into `events`). Pushing here as
|
|
348
|
+
// well would write the same `agent.event` twice on the socket, breaking the
|
|
349
|
+
// "exactly once" delivery contract. The retry-exhausted failed branch in
|
|
350
|
+
// `#retryWaitingPlanRow` keeps its own push because no caller sees its result.
|
|
351
|
+
return result;
|
|
251
352
|
}
|
|
252
353
|
async drainPendingPlans() {
|
|
253
354
|
if (!this.#stores.statusEventPlans)
|
|
@@ -278,51 +379,192 @@ export class AgentIndexService {
|
|
|
278
379
|
}
|
|
279
380
|
});
|
|
280
381
|
// Individual rows may reject (after markRetry) but the drain itself never
|
|
281
|
-
// rejects: every row is either drained, cancelled, or retried.
|
|
282
|
-
|
|
382
|
+
// rejects: every row is either drained, cancelled, or retried. A rejected
|
|
383
|
+
// row is still an anomaly (its own drain path threw), so it is reported
|
|
384
|
+
// instead of being silently folded into the settled results.
|
|
385
|
+
const results = await Promise.allSettled(tasks);
|
|
386
|
+
const rejected = results.flatMap((result) => result.status === "rejected" ? [result.reason] : []);
|
|
387
|
+
if (rejected.length > 0) {
|
|
388
|
+
console.error("Herdsman status event plan drain rejected rows", {
|
|
389
|
+
rejected: rejected.length,
|
|
390
|
+
total: results.length,
|
|
391
|
+
errors: rejected.map((reason) => reason instanceof Error ? reason.message : String(reason)),
|
|
392
|
+
});
|
|
393
|
+
}
|
|
283
394
|
await this.#backfillFailedPlanEvents();
|
|
395
|
+
await this.#backfillDiscardedPlanEvents();
|
|
284
396
|
}
|
|
285
397
|
async #backfillFailedPlanEvents() {
|
|
286
398
|
const store = this.#stores.statusEventPlans;
|
|
287
399
|
if (!store)
|
|
288
400
|
return;
|
|
401
|
+
let backfilled = 0;
|
|
402
|
+
let skipped = 0;
|
|
289
403
|
for (const row of store.listFailed()) {
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
continue;
|
|
295
|
-
let agent;
|
|
404
|
+
// Per-row isolation: one unresolvable or unwritable row must not abort the
|
|
405
|
+
// whole backfill round. The legacy loop had no guard, so the first
|
|
406
|
+
// `FOREIGN KEY constraint failed` threw out of `drainPendingPlans` and no
|
|
407
|
+
// later row was ever attempted.
|
|
296
408
|
try {
|
|
297
|
-
|
|
409
|
+
if (row.lastError === PLAN_WAITING_HISTORY) {
|
|
410
|
+
skipped += 1;
|
|
411
|
+
continue;
|
|
412
|
+
}
|
|
413
|
+
const existing = this.#stores.sqlite
|
|
414
|
+
.prepare("select 1 from agent_events where herdr_session_name = ? and idempotency_key = ?")
|
|
415
|
+
.get(row.herdrSessionName, `agent.failed:plan:${row.id}`);
|
|
416
|
+
if (existing) {
|
|
417
|
+
skipped += 1;
|
|
418
|
+
continue;
|
|
419
|
+
}
|
|
420
|
+
const agent = this.#resolveBackfillAgent(row);
|
|
421
|
+
if (!agent) {
|
|
422
|
+
skipped += 1;
|
|
423
|
+
continue;
|
|
424
|
+
}
|
|
425
|
+
const event = this.#appendPlanFailedEvent({
|
|
426
|
+
agent,
|
|
427
|
+
attempts: row.attempts,
|
|
428
|
+
compactHistory: row.compactHistory,
|
|
429
|
+
from: row.fromStatus,
|
|
430
|
+
planCreatedAt: row.createdAt.getTime(),
|
|
431
|
+
planId: row.id,
|
|
432
|
+
reason: row.lastError,
|
|
433
|
+
to: row.toStatus,
|
|
434
|
+
});
|
|
435
|
+
// Stock terminal results are ingested silently: the row is written, then
|
|
436
|
+
// immediately acknowledged/non-deliverable, so it stays queryable without
|
|
437
|
+
// waking the current owner with a long-past failure.
|
|
438
|
+
this.#stores.agentEvents.markAcked(event.id);
|
|
439
|
+
backfilled += 1;
|
|
298
440
|
}
|
|
299
|
-
catch {
|
|
300
|
-
|
|
441
|
+
catch (error) {
|
|
442
|
+
skipped += 1;
|
|
443
|
+
console.warn("Herdsman skipping agent.failed backfill row", {
|
|
444
|
+
agentId: row.agentId,
|
|
445
|
+
herdrSessionName: row.herdrSessionName,
|
|
446
|
+
paneId: row.paneId,
|
|
447
|
+
planId: row.id,
|
|
448
|
+
error: error instanceof Error ? error.message : String(error),
|
|
449
|
+
});
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
if (backfilled > 0 || skipped > 0) {
|
|
453
|
+
console.info("Herdsman backfilled agent.failed rows", {
|
|
454
|
+
backfilled,
|
|
455
|
+
skipped,
|
|
456
|
+
total: backfilled + skipped,
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
async #backfillDiscardedPlanEvents() {
|
|
461
|
+
const store = this.#stores.statusEventPlans;
|
|
462
|
+
if (!store)
|
|
463
|
+
return;
|
|
464
|
+
let backfilled = 0;
|
|
465
|
+
let skipped = 0;
|
|
466
|
+
for (const row of store.listDiscarded()) {
|
|
467
|
+
try {
|
|
468
|
+
const existing = this.#stores.sqlite
|
|
469
|
+
.prepare("select 1 from agent_events where herdr_session_name = ? and idempotency_key = ?")
|
|
470
|
+
.get(row.herdrSessionName, `agent.discarded:plan:${row.id}`);
|
|
471
|
+
if (existing) {
|
|
472
|
+
skipped += 1;
|
|
473
|
+
continue;
|
|
474
|
+
}
|
|
475
|
+
const agent = this.#resolveBackfillAgent(row);
|
|
301
476
|
if (!agent) {
|
|
302
|
-
|
|
303
|
-
agentId: row.agentId,
|
|
304
|
-
herdrSessionName: row.herdrSessionName,
|
|
305
|
-
planId: row.id,
|
|
306
|
-
});
|
|
477
|
+
skipped += 1;
|
|
307
478
|
continue;
|
|
308
479
|
}
|
|
480
|
+
const event = this.#appendPlanDiscardedEvent({
|
|
481
|
+
agent,
|
|
482
|
+
attempts: row.attempts,
|
|
483
|
+
compactHistory: row.compactHistory,
|
|
484
|
+
from: row.fromStatus,
|
|
485
|
+
planCreatedAt: row.createdAt.getTime(),
|
|
486
|
+
planId: row.id,
|
|
487
|
+
reason: row.lastError,
|
|
488
|
+
to: row.toStatus,
|
|
489
|
+
});
|
|
490
|
+
// Same silent-ingest contract as the failed backfill.
|
|
491
|
+
this.#stores.agentEvents.markAcked(event.id);
|
|
492
|
+
backfilled += 1;
|
|
309
493
|
}
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
494
|
+
catch (error) {
|
|
495
|
+
skipped += 1;
|
|
496
|
+
console.warn("Herdsman skipping agent.discarded backfill row", {
|
|
497
|
+
agentId: row.agentId,
|
|
498
|
+
herdrSessionName: row.herdrSessionName,
|
|
499
|
+
paneId: row.paneId,
|
|
500
|
+
planId: row.id,
|
|
501
|
+
error: error instanceof Error ? error.message : String(error),
|
|
502
|
+
});
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
if (backfilled > 0 || skipped > 0) {
|
|
506
|
+
console.info("Herdsman backfilled agent.discarded rows", {
|
|
507
|
+
backfilled,
|
|
508
|
+
skipped,
|
|
509
|
+
total: backfilled + skipped,
|
|
318
510
|
});
|
|
319
511
|
}
|
|
320
512
|
}
|
|
321
|
-
|
|
513
|
+
/**
|
|
514
|
+
* Resolves the metadata a terminal plan outcome may use to reach an
|
|
515
|
+
* orchestrator: the live `agents` row when it still exists, otherwise a
|
|
516
|
+
* minimal record stitched from the pane's own event history.
|
|
517
|
+
*/
|
|
518
|
+
#resolveBackfillAgent(row) {
|
|
519
|
+
try {
|
|
520
|
+
return this.#stores.agents.get(row.agentId);
|
|
521
|
+
}
|
|
522
|
+
catch {
|
|
523
|
+
const stitched = this.#minimalAgentFromLatestEvent({
|
|
524
|
+
agentId: row.agentId,
|
|
525
|
+
createdBefore: row.createdAt.getTime(),
|
|
526
|
+
herdrSessionName: row.herdrSessionName,
|
|
527
|
+
paneGeneration: row.paneGeneration,
|
|
528
|
+
paneId: row.paneId,
|
|
529
|
+
});
|
|
530
|
+
if (!stitched) {
|
|
531
|
+
console.warn("Herdsman skipping terminal event backfill; no agent row or pane events", {
|
|
532
|
+
agentId: row.agentId,
|
|
533
|
+
herdrSessionName: row.herdrSessionName,
|
|
534
|
+
paneId: row.paneId,
|
|
535
|
+
planId: row.id,
|
|
536
|
+
});
|
|
537
|
+
}
|
|
538
|
+
return stitched;
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
/**
|
|
542
|
+
* Reconstructs the minimum agent metadata a terminal event needs from the
|
|
543
|
+
* pane's own event history.
|
|
544
|
+
*
|
|
545
|
+
* The lookup is keyed by `(herdr_session_name, pane_id)` plus the plan's
|
|
546
|
+
* `pane_generation`, and is bounded by `createdBefore` (the plan row's
|
|
547
|
+
* `created_at`). Matching on `agent_id` alone cannot work here: physical
|
|
548
|
+
* deletion of the `agents` row runs `ON DELETE SET NULL` over every event, so
|
|
549
|
+
* the legacy stitch always came back empty, and a pane-id-only match would
|
|
550
|
+
* happily stitch an old failure onto a newer instance of the same pane. The
|
|
551
|
+
* generation and time bounds are what keep the two apart.
|
|
552
|
+
*/
|
|
553
|
+
#minimalAgentFromLatestEvent(input) {
|
|
554
|
+
const generationClause = input.paneGeneration === null ? "pane_generation is null" : "pane_generation = ?";
|
|
555
|
+
const params = [
|
|
556
|
+
input.herdrSessionName,
|
|
557
|
+
input.paneId,
|
|
558
|
+
input.createdBefore,
|
|
559
|
+
];
|
|
560
|
+
if (input.paneGeneration !== null)
|
|
561
|
+
params.push(input.paneGeneration);
|
|
322
562
|
const latest = this.#stores.sqlite
|
|
323
563
|
.prepare(`select pane_id, pane_generation, terminal_id, workspace_id, herdr_session_name
|
|
324
|
-
from agent_events
|
|
325
|
-
|
|
564
|
+
from agent_events
|
|
565
|
+
where herdr_session_name = ? and pane_id = ? and created_at <= ? and ${generationClause}
|
|
566
|
+
order by id desc limit 1`)
|
|
567
|
+
.get(...params);
|
|
326
568
|
if (!latest?.pane_id || !latest.workspace_id)
|
|
327
569
|
return undefined;
|
|
328
570
|
return {
|
|
@@ -333,8 +575,8 @@ export class AgentIndexService {
|
|
|
333
575
|
firstSeenAt: new Date(0),
|
|
334
576
|
focused: false,
|
|
335
577
|
foregroundCwd: null,
|
|
336
|
-
herdrSessionName: latest.herdr_session_name || herdrSessionName,
|
|
337
|
-
id: agentId,
|
|
578
|
+
herdrSessionName: latest.herdr_session_name || input.herdrSessionName,
|
|
579
|
+
id: input.agentId,
|
|
338
580
|
lastSeenAt: new Date(0),
|
|
339
581
|
name: null,
|
|
340
582
|
paneId: latest.pane_id,
|
|
@@ -346,30 +588,50 @@ export class AgentIndexService {
|
|
|
346
588
|
};
|
|
347
589
|
}
|
|
348
590
|
async #drainPlanRow(row) {
|
|
591
|
+
const store = this.#stores.statusEventPlans;
|
|
592
|
+
if (!store)
|
|
593
|
+
return;
|
|
594
|
+
let current;
|
|
595
|
+
try {
|
|
596
|
+
current = runSqliteTransaction(this.#stores.sqlite, () => {
|
|
597
|
+
const candidate = store.get(row.id);
|
|
598
|
+
if (candidate.status !== "pending")
|
|
599
|
+
return null;
|
|
600
|
+
return candidate;
|
|
601
|
+
});
|
|
602
|
+
}
|
|
603
|
+
catch {
|
|
604
|
+
this.#clearWaitingTimer(row.id);
|
|
605
|
+
return;
|
|
606
|
+
}
|
|
607
|
+
if (!current) {
|
|
608
|
+
this.#clearWaitingTimer(row.id);
|
|
609
|
+
return;
|
|
610
|
+
}
|
|
349
611
|
let agent;
|
|
350
612
|
try {
|
|
351
613
|
agent =
|
|
352
614
|
this.#stores.agents.findByPane({
|
|
353
|
-
herdrSessionName:
|
|
354
|
-
paneId:
|
|
355
|
-
paneGeneration:
|
|
356
|
-
}) ?? this.#stores.agents.get(
|
|
615
|
+
herdrSessionName: current.herdrSessionName,
|
|
616
|
+
paneId: current.paneId,
|
|
617
|
+
paneGeneration: current.paneGeneration,
|
|
618
|
+
}) ?? this.#stores.agents.get(current.agentId);
|
|
357
619
|
}
|
|
358
620
|
catch {
|
|
359
621
|
agent = undefined;
|
|
360
622
|
}
|
|
361
623
|
if (!agent) {
|
|
362
|
-
|
|
624
|
+
store.markCancelled(current.id);
|
|
363
625
|
console.warn("Herdsman cancelling status event plan for missing agent", {
|
|
364
|
-
agentId:
|
|
365
|
-
herdrSessionName:
|
|
366
|
-
paneId:
|
|
367
|
-
from:
|
|
368
|
-
to:
|
|
626
|
+
agentId: current.agentId,
|
|
627
|
+
herdrSessionName: current.herdrSessionName,
|
|
628
|
+
paneId: current.paneId,
|
|
629
|
+
from: current.fromStatus,
|
|
630
|
+
to: current.toStatus,
|
|
369
631
|
});
|
|
370
632
|
return;
|
|
371
633
|
}
|
|
372
|
-
let activeCompact =
|
|
634
|
+
let activeCompact = current.compactHistory;
|
|
373
635
|
try {
|
|
374
636
|
const refreshed = await this.#context.refreshAgent({
|
|
375
637
|
agent,
|
|
@@ -379,52 +641,88 @@ export class AgentIndexService {
|
|
|
379
641
|
activeCompact = refreshed.snapshot.compactHistory;
|
|
380
642
|
}
|
|
381
643
|
catch (err) {
|
|
382
|
-
|
|
383
|
-
const updated =
|
|
644
|
+
const retryError = current.lastError === "degraded" ? new Error("degraded") : new PlanWaitingHistoryError();
|
|
645
|
+
const updated = store.markRetry(current.id, retryError);
|
|
646
|
+
if (!updated) {
|
|
647
|
+
this.#clearWaitingTimer(current.id);
|
|
648
|
+
return;
|
|
649
|
+
}
|
|
384
650
|
if (updated.status === "pending") {
|
|
385
651
|
console.warn("Herdsman failed to refresh agent during drain plan row, keeping waiting", {
|
|
386
|
-
planId:
|
|
652
|
+
planId: current.id,
|
|
387
653
|
agentId: agent.id,
|
|
388
654
|
error: err,
|
|
389
655
|
});
|
|
390
|
-
|
|
656
|
+
// Invariant: the ring is only re-armed by a genuine
|
|
657
|
+
// PlanWaitingHistoryError; a degraded row stays pending for the next
|
|
658
|
+
// refresh to drain instead of spinning this callback again.
|
|
659
|
+
if (retryRingAuthorized(retryError)) {
|
|
660
|
+
this.#scheduleWaitingHistoryRetry(current.id, {
|
|
661
|
+
agent,
|
|
662
|
+
compactHistory: current.compactHistory,
|
|
663
|
+
from: current.fromStatus,
|
|
664
|
+
to: current.toStatus,
|
|
665
|
+
...(current.herdrEventKey ? { herdrEventKey: current.herdrEventKey } : {}),
|
|
666
|
+
});
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
else if (updated.status === "discarded") {
|
|
670
|
+
this.#clearWaitingTimer(current.id);
|
|
671
|
+
console.info("Herdsman plan marked discarded during drain", {
|
|
672
|
+
planId: current.id,
|
|
673
|
+
agentId: agent.id,
|
|
674
|
+
from: current.fromStatus,
|
|
675
|
+
to: current.toStatus,
|
|
676
|
+
reason: updated.lastError,
|
|
677
|
+
});
|
|
678
|
+
this.#appendPlanDiscardedEvent({
|
|
391
679
|
agent,
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
680
|
+
attempts: updated.attempts,
|
|
681
|
+
compactHistory: current.compactHistory,
|
|
682
|
+
from: current.fromStatus,
|
|
683
|
+
planCreatedAt: current.createdAt.getTime(),
|
|
684
|
+
planId: current.id,
|
|
685
|
+
reason: updated.lastError,
|
|
686
|
+
to: current.toStatus,
|
|
396
687
|
});
|
|
397
688
|
}
|
|
398
689
|
else if (updated.status === "failed") {
|
|
399
|
-
this.#clearWaitingTimer(
|
|
690
|
+
this.#clearWaitingTimer(current.id);
|
|
400
691
|
console.warn("Herdsman plan marked failed", {
|
|
401
|
-
planId:
|
|
692
|
+
planId: current.id,
|
|
402
693
|
agentId: agent.id,
|
|
403
|
-
from:
|
|
404
|
-
to:
|
|
694
|
+
from: current.fromStatus,
|
|
695
|
+
to: current.toStatus,
|
|
405
696
|
reason: updated.lastError,
|
|
406
697
|
});
|
|
407
|
-
this.#appendPlanFailedEvent({
|
|
698
|
+
const failedEvent = this.#appendPlanFailedEvent({
|
|
408
699
|
agent,
|
|
409
700
|
attempts: updated.attempts,
|
|
410
|
-
compactHistory:
|
|
411
|
-
from:
|
|
412
|
-
|
|
701
|
+
compactHistory: current.compactHistory,
|
|
702
|
+
from: current.fromStatus,
|
|
703
|
+
planCreatedAt: current.createdAt.getTime(),
|
|
704
|
+
planId: current.id,
|
|
413
705
|
reason: updated.lastError,
|
|
414
|
-
to:
|
|
706
|
+
to: current.toStatus,
|
|
415
707
|
});
|
|
708
|
+
if (VALID_AGENT_EVENT_TYPES.has(failedEvent.type)) {
|
|
709
|
+
this.#onAgentEvent?.(failedEvent);
|
|
710
|
+
}
|
|
416
711
|
}
|
|
417
712
|
return;
|
|
418
713
|
}
|
|
419
714
|
const plan = {
|
|
420
715
|
agent,
|
|
421
|
-
attempts:
|
|
716
|
+
attempts: current.attempts,
|
|
422
717
|
compactHistory: activeCompact,
|
|
423
|
-
from:
|
|
424
|
-
to:
|
|
425
|
-
...(
|
|
718
|
+
from: current.fromStatus,
|
|
719
|
+
to: current.toStatus,
|
|
720
|
+
...(current.herdrEventKey ? { herdrEventKey: current.herdrEventKey } : {}),
|
|
426
721
|
};
|
|
427
|
-
await this.#runPlanRow(
|
|
722
|
+
const event = await this.#runPlanRow(current, plan);
|
|
723
|
+
if (event && VALID_AGENT_EVENT_TYPES.has(event.type)) {
|
|
724
|
+
this.#onAgentEvent?.(event);
|
|
725
|
+
}
|
|
428
726
|
}
|
|
429
727
|
#enqueueAgentPlan(agentId, operation) {
|
|
430
728
|
const previous = this.#planTailByAgent.get(agentId) ?? Promise.resolve();
|
|
@@ -445,14 +743,21 @@ export class AgentIndexService {
|
|
|
445
743
|
const current = store.get(row.id);
|
|
446
744
|
if (current.status === "completed" ||
|
|
447
745
|
current.status === "cancelled" ||
|
|
448
|
-
current.status === "failed"
|
|
746
|
+
current.status === "failed" ||
|
|
747
|
+
current.status === "discarded") {
|
|
748
|
+
this.#clearWaitingTimer(row.id);
|
|
749
|
+
return undefined;
|
|
750
|
+
}
|
|
751
|
+
if (!store.markRunning(row.id)) {
|
|
449
752
|
this.#clearWaitingTimer(row.id);
|
|
450
753
|
return undefined;
|
|
451
754
|
}
|
|
452
|
-
store.markRunning(row.id);
|
|
453
755
|
let activePlan = {
|
|
454
756
|
...plan,
|
|
455
757
|
attempts: row.attempts,
|
|
758
|
+
// Row correlation for observability (see `#logDegradedRelease`): the plan
|
|
759
|
+
// id is only known here, where the store row is in hand.
|
|
760
|
+
planId: row.id,
|
|
456
761
|
};
|
|
457
762
|
if (row.attempts > 0) {
|
|
458
763
|
try {
|
|
@@ -465,6 +770,7 @@ export class AgentIndexService {
|
|
|
465
770
|
...plan,
|
|
466
771
|
attempts: row.attempts,
|
|
467
772
|
compactHistory: refreshed.snapshot.compactHistory,
|
|
773
|
+
planId: row.id,
|
|
468
774
|
};
|
|
469
775
|
}
|
|
470
776
|
catch (err) {
|
|
@@ -473,13 +779,45 @@ export class AgentIndexService {
|
|
|
473
779
|
agentId: plan.agent.id,
|
|
474
780
|
error: err,
|
|
475
781
|
});
|
|
476
|
-
|
|
477
|
-
|
|
782
|
+
// Keep the degraded reason when the plan was already degraded by an
|
|
783
|
+
// earlier round: markRetry decides discarded (silent) vs failed (wakes)
|
|
784
|
+
// from this message, so passing PLAN_WAITING_HISTORY here would let a
|
|
785
|
+
// degraded plan exhaust silently.
|
|
786
|
+
const retryError = row.lastError === "degraded" ? new Error("degraded") : new PlanWaitingHistoryError();
|
|
787
|
+
const updated = store.markRetry(row.id, retryError);
|
|
788
|
+
if (!updated) {
|
|
789
|
+
this.#clearWaitingTimer(row.id);
|
|
790
|
+
return undefined;
|
|
791
|
+
}
|
|
478
792
|
if (updated.status === "pending") {
|
|
479
|
-
|
|
793
|
+
// Invariant: only a genuine PlanWaitingHistoryError authorizes the
|
|
794
|
+
// ring; a degraded pre-refresh round records the synthetic reason and
|
|
795
|
+
// waits for the next refresh instead.
|
|
796
|
+
if (retryRingAuthorized(retryError))
|
|
797
|
+
this.#scheduleWaitingHistoryRetry(row.id, plan);
|
|
480
798
|
}
|
|
481
799
|
else {
|
|
482
800
|
this.#clearWaitingTimer(row.id);
|
|
801
|
+
if (updated.status === "discarded") {
|
|
802
|
+
console.info("Herdsman status event plan discarded after max attempts", {
|
|
803
|
+
planId: row.id,
|
|
804
|
+
agentId: row.agentId,
|
|
805
|
+
attempts: updated.attempts,
|
|
806
|
+
from: row.fromStatus,
|
|
807
|
+
to: row.toStatus,
|
|
808
|
+
herdrSessionName: row.herdrSessionName,
|
|
809
|
+
});
|
|
810
|
+
return this.#appendPlanDiscardedEvent({
|
|
811
|
+
agent: plan.agent,
|
|
812
|
+
attempts: updated.attempts,
|
|
813
|
+
compactHistory: plan.compactHistory,
|
|
814
|
+
from: row.fromStatus,
|
|
815
|
+
planCreatedAt: row.createdAt.getTime(),
|
|
816
|
+
planId: row.id,
|
|
817
|
+
reason: updated.lastError,
|
|
818
|
+
to: row.toStatus,
|
|
819
|
+
});
|
|
820
|
+
}
|
|
483
821
|
if (updated.status === "failed") {
|
|
484
822
|
console.warn("Herdsman status event plan failed after max attempts", {
|
|
485
823
|
planId: row.id,
|
|
@@ -494,6 +832,7 @@ export class AgentIndexService {
|
|
|
494
832
|
attempts: updated.attempts,
|
|
495
833
|
compactHistory: plan.compactHistory,
|
|
496
834
|
from: row.fromStatus,
|
|
835
|
+
planCreatedAt: row.createdAt.getTime(),
|
|
497
836
|
planId: row.id,
|
|
498
837
|
reason: updated.lastError,
|
|
499
838
|
to: row.toStatus,
|
|
@@ -521,18 +860,100 @@ export class AgentIndexService {
|
|
|
521
860
|
return undefined;
|
|
522
861
|
}
|
|
523
862
|
this.#clearWaitingTimer(row.id);
|
|
863
|
+
const eventPayload = event?.payload;
|
|
864
|
+
const isDegraded = !!event &&
|
|
865
|
+
typeof eventPayload === "object" &&
|
|
866
|
+
eventPayload !== null &&
|
|
867
|
+
eventPayload.degraded === true;
|
|
868
|
+
if (isDegraded) {
|
|
869
|
+
if (event) {
|
|
870
|
+
this.#stores.agentEvents.invalidateById(event.id, "degraded_retry");
|
|
871
|
+
Object.assign(event, { status: "invalidated", deliverable: 0 });
|
|
872
|
+
}
|
|
873
|
+
const degradedError = new Error("degraded");
|
|
874
|
+
const updated = store.markRetry(row.id, degradedError);
|
|
875
|
+
if (!updated) {
|
|
876
|
+
return undefined;
|
|
877
|
+
}
|
|
878
|
+
if (updated.status === "pending") {
|
|
879
|
+
// Invariant: `degraded` is a synthetic reason, so it never authorizes
|
|
880
|
+
// the retry ring; the row stays pending and the next refresh
|
|
881
|
+
// re-drains it.
|
|
882
|
+
if (retryRingAuthorized(degradedError))
|
|
883
|
+
this.#scheduleWaitingHistoryRetry(row.id, plan);
|
|
884
|
+
return undefined;
|
|
885
|
+
}
|
|
886
|
+
else {
|
|
887
|
+
this.#clearWaitingTimer(row.id);
|
|
888
|
+
if (updated.status === "discarded") {
|
|
889
|
+
return this.#appendPlanDiscardedEvent({
|
|
890
|
+
agent: plan.agent,
|
|
891
|
+
attempts: updated.attempts,
|
|
892
|
+
compactHistory: activePlan.compactHistory,
|
|
893
|
+
from: row.fromStatus,
|
|
894
|
+
planCreatedAt: row.createdAt.getTime(),
|
|
895
|
+
planId: row.id,
|
|
896
|
+
reason: updated.lastError,
|
|
897
|
+
to: row.toStatus,
|
|
898
|
+
});
|
|
899
|
+
}
|
|
900
|
+
if (updated.status === "failed") {
|
|
901
|
+
return this.#appendPlanFailedEvent({
|
|
902
|
+
agent: plan.agent,
|
|
903
|
+
attempts: updated.attempts,
|
|
904
|
+
compactHistory: activePlan.compactHistory,
|
|
905
|
+
from: row.fromStatus,
|
|
906
|
+
planCreatedAt: row.createdAt.getTime(),
|
|
907
|
+
planId: row.id,
|
|
908
|
+
reason: updated.lastError,
|
|
909
|
+
to: row.toStatus,
|
|
910
|
+
});
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
return undefined;
|
|
914
|
+
}
|
|
524
915
|
store.markCompleted(row.id);
|
|
525
916
|
return event;
|
|
526
917
|
}
|
|
527
918
|
catch (error) {
|
|
528
919
|
if (error instanceof PlanWaitingHistoryError) {
|
|
529
|
-
|
|
530
|
-
const updated = store.
|
|
920
|
+
const retryError = row.lastError === "degraded" ? new Error("degraded") : error;
|
|
921
|
+
const updated = store.markRetry(row.id, retryError);
|
|
922
|
+
if (!updated) {
|
|
923
|
+
this.#clearWaitingTimer(row.id);
|
|
924
|
+
return undefined;
|
|
925
|
+
}
|
|
531
926
|
if (updated.status === "pending") {
|
|
532
|
-
|
|
927
|
+
// Invariant: only a genuine PlanWaitingHistoryError authorizes the
|
|
928
|
+
// ring. A row that already carries the synthetic `degraded` reason is
|
|
929
|
+
// recorded as degraded again and waits for the next refresh instead of
|
|
930
|
+
// re-arming the 10s timer, so this catch cannot be used as a back door
|
|
931
|
+
// that lets a degraded round keep the ring alive.
|
|
932
|
+
if (retryRingAuthorized(retryError))
|
|
933
|
+
this.#scheduleWaitingHistoryRetry(row.id, plan);
|
|
533
934
|
}
|
|
534
935
|
else {
|
|
535
936
|
this.#clearWaitingTimer(row.id);
|
|
937
|
+
if (updated.status === "discarded") {
|
|
938
|
+
console.info("Herdsman status event plan discarded after max attempts", {
|
|
939
|
+
planId: row.id,
|
|
940
|
+
agentId: row.agentId,
|
|
941
|
+
attempts: updated.attempts,
|
|
942
|
+
from: row.fromStatus,
|
|
943
|
+
to: row.toStatus,
|
|
944
|
+
herdrSessionName: row.herdrSessionName,
|
|
945
|
+
});
|
|
946
|
+
return this.#appendPlanDiscardedEvent({
|
|
947
|
+
agent: plan.agent,
|
|
948
|
+
attempts: updated.attempts,
|
|
949
|
+
compactHistory: activePlan.compactHistory,
|
|
950
|
+
from: row.fromStatus,
|
|
951
|
+
planCreatedAt: row.createdAt.getTime(),
|
|
952
|
+
planId: row.id,
|
|
953
|
+
reason: updated.lastError,
|
|
954
|
+
to: row.toStatus,
|
|
955
|
+
});
|
|
956
|
+
}
|
|
536
957
|
if (updated.status === "failed") {
|
|
537
958
|
console.warn("Herdsman status event plan failed after max attempts", {
|
|
538
959
|
planId: row.id,
|
|
@@ -547,6 +968,7 @@ export class AgentIndexService {
|
|
|
547
968
|
attempts: updated.attempts,
|
|
548
969
|
compactHistory: activePlan.compactHistory,
|
|
549
970
|
from: row.fromStatus,
|
|
971
|
+
planCreatedAt: row.createdAt.getTime(),
|
|
550
972
|
planId: row.id,
|
|
551
973
|
reason: updated.lastError,
|
|
552
974
|
to: row.toStatus,
|
|
@@ -569,8 +991,35 @@ export class AgentIndexService {
|
|
|
569
991
|
return undefined;
|
|
570
992
|
}
|
|
571
993
|
this.#clearWaitingTimer(row.id);
|
|
572
|
-
|
|
573
|
-
const updated = store.
|
|
994
|
+
const retryError = row.lastError === "degraded" ? new Error("degraded") : error;
|
|
995
|
+
const updated = store.markRetry(row.id, retryError);
|
|
996
|
+
if (!updated) {
|
|
997
|
+
return undefined;
|
|
998
|
+
}
|
|
999
|
+
// Invariant: only a genuine PlanWaitingHistoryError re-arms the retry ring,
|
|
1000
|
+
// and that case was handled above. On this generic path `retryError` is
|
|
1001
|
+
// either the synthetic `degraded` reason or an unrelated failure, so no
|
|
1002
|
+
// spin is scheduled: a still-pending row is drained by the next refresh.
|
|
1003
|
+
if (updated.status === "discarded") {
|
|
1004
|
+
console.info("Herdsman status event plan discarded after max attempts", {
|
|
1005
|
+
planId: row.id,
|
|
1006
|
+
agentId: row.agentId,
|
|
1007
|
+
attempts: updated.attempts,
|
|
1008
|
+
from: row.fromStatus,
|
|
1009
|
+
to: row.toStatus,
|
|
1010
|
+
herdrSessionName: row.herdrSessionName,
|
|
1011
|
+
});
|
|
1012
|
+
return this.#appendPlanDiscardedEvent({
|
|
1013
|
+
agent: plan.agent,
|
|
1014
|
+
attempts: updated.attempts,
|
|
1015
|
+
compactHistory: activePlan.compactHistory,
|
|
1016
|
+
from: row.fromStatus,
|
|
1017
|
+
planCreatedAt: row.createdAt.getTime(),
|
|
1018
|
+
planId: row.id,
|
|
1019
|
+
reason: updated.lastError,
|
|
1020
|
+
to: row.toStatus,
|
|
1021
|
+
});
|
|
1022
|
+
}
|
|
574
1023
|
if (updated.status === "failed") {
|
|
575
1024
|
console.warn("Herdsman status event plan failed after max attempts", {
|
|
576
1025
|
planId: row.id,
|
|
@@ -585,6 +1034,7 @@ export class AgentIndexService {
|
|
|
585
1034
|
attempts: updated.attempts,
|
|
586
1035
|
compactHistory: activePlan.compactHistory,
|
|
587
1036
|
from: row.fromStatus,
|
|
1037
|
+
planCreatedAt: row.createdAt.getTime(),
|
|
588
1038
|
planId: row.id,
|
|
589
1039
|
reason: updated.lastError,
|
|
590
1040
|
to: row.toStatus,
|
|
@@ -595,6 +1045,12 @@ export class AgentIndexService {
|
|
|
595
1045
|
}
|
|
596
1046
|
async #waitForHistoryAdvance(input) {
|
|
597
1047
|
const maxAttempts = input.maxAttempts ?? 8;
|
|
1048
|
+
// Fixed 200ms interval, no exponential backoff: the history cache either
|
|
1049
|
+
// advances shortly after the turn signal or it does not, and a doubling
|
|
1050
|
+
// delay only pushed the first deliverable terminal event tens of seconds
|
|
1051
|
+
// away. The whole window is hard bounded by maxTotalMs.
|
|
1052
|
+
const baseDelay = 200;
|
|
1053
|
+
const maxTotalMs = 1500;
|
|
598
1054
|
let refreshed = input.initial
|
|
599
1055
|
? { snapshot: { compactHistory: input.initial } }
|
|
600
1056
|
: await this.#context.refreshAgent({
|
|
@@ -602,20 +1058,34 @@ export class AgentIndexService {
|
|
|
602
1058
|
forceRefresh: true,
|
|
603
1059
|
identityChanged: false,
|
|
604
1060
|
});
|
|
1061
|
+
const start = this.#now();
|
|
605
1062
|
for (let attempt = 0; attempt < maxAttempts &&
|
|
606
1063
|
!historyHasAdvanced(refreshed.snapshot.compactHistory, input.baseline, {
|
|
607
1064
|
requireAssistantChange: input.requireAssistantChange,
|
|
608
1065
|
}) &&
|
|
609
|
-
!input.controller.signal.aborted
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
1066
|
+
!input.controller.signal.aborted &&
|
|
1067
|
+
this.#now() - start < maxTotalMs; attempt += 1) {
|
|
1068
|
+
// Clamp the last step to the remaining budget so the total wait never
|
|
1069
|
+
// exceeds maxTotalMs even though the loop condition is checked before
|
|
1070
|
+
// sleeping.
|
|
1071
|
+
const remaining = maxTotalMs - (this.#now() - start);
|
|
1072
|
+
if (remaining <= 0)
|
|
1073
|
+
break;
|
|
1074
|
+
await this.#sleep(Math.min(baseDelay, remaining), input.controller.signal);
|
|
1075
|
+
if (input.controller.signal.aborted) {
|
|
1076
|
+
console.debug("Herdsman retrying status event plan because wait was aborted during history refresh", {
|
|
1077
|
+
agentId: input.agent.id,
|
|
613
1078
|
herdrSessionName: input.agent.herdrSessionName,
|
|
614
1079
|
paneId: input.agent.paneId,
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
1080
|
+
});
|
|
1081
|
+
throw new PlanWaitingHistoryError();
|
|
1082
|
+
}
|
|
1083
|
+
if (this.#stores.agents.isPaneClosed({
|
|
1084
|
+
herdrSessionName: input.agent.herdrSessionName,
|
|
1085
|
+
paneId: input.agent.paneId,
|
|
1086
|
+
paneGeneration: input.agent.paneGeneration ?? null,
|
|
1087
|
+
})) {
|
|
1088
|
+
console.debug("Herdsman skipping status event generation because pane closed during history refresh", {
|
|
619
1089
|
agentId: input.agent.id,
|
|
620
1090
|
herdrSessionName: input.agent.herdrSessionName,
|
|
621
1091
|
paneId: input.agent.paneId,
|
|
@@ -981,8 +1451,23 @@ export class AgentIndexService {
|
|
|
981
1451
|
paneGeneration: paneGenerationOf(agent),
|
|
982
1452
|
});
|
|
983
1453
|
}
|
|
1454
|
+
/**
|
|
1455
|
+
* Distinguishes the two reasons an in-flight wait can be aborted. A closed pane
|
|
1456
|
+
* makes the transition moot, so the plan is skipped; a daemon shutdown leaves
|
|
1457
|
+
* the pane alive, so the plan must stay retryable instead of being completed
|
|
1458
|
+
* without ever emitting the transition (it is drained on the next start).
|
|
1459
|
+
*/
|
|
1460
|
+
#throwIfShutdownAbort(controller) {
|
|
1461
|
+
if (controller.signal.aborted && this.#shutdownSignal?.aborted) {
|
|
1462
|
+
throw new PlanWaitingHistoryError();
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
984
1465
|
#registerActiveWaiter(agent, controller) {
|
|
985
1466
|
const key = `${agent.herdrSessionName}\0${agent.paneId}\0${agent.paneGeneration ?? ""}`;
|
|
1467
|
+
// A wait that starts after shutdown already aborted must not sleep at all:
|
|
1468
|
+
// abort it up front so the caller short-circuits like a closed pane.
|
|
1469
|
+
if (this.#shutdownSignal?.aborted)
|
|
1470
|
+
controller.abort();
|
|
986
1471
|
let set = this.#activeWaiters.get(key);
|
|
987
1472
|
if (!set) {
|
|
988
1473
|
set = new Set();
|
|
@@ -1010,10 +1495,36 @@ export class AgentIndexService {
|
|
|
1010
1495
|
}
|
|
1011
1496
|
}
|
|
1012
1497
|
}
|
|
1498
|
+
/**
|
|
1499
|
+
* Observability for a degraded release. The emitted status event carries the
|
|
1500
|
+
* specific `degradedReason`, but the plan row itself only stores the synthetic
|
|
1501
|
+
* `degraded` marker (`row.lastError`), so without this line the cause is
|
|
1502
|
+
* invisible in the daemon log. It cannot ride along in the failed-event payload
|
|
1503
|
+
* either: for most append sites only `row.lastError` is at hand, so handing them
|
|
1504
|
+
* the specific reason would mean persisting a new plan field (schema change).
|
|
1505
|
+
* That is not universal — at least three call sites in this file already have the
|
|
1506
|
+
* specific reason in their own call stack (`:1074` the status event payload's
|
|
1507
|
+
* `degradedReason`, `:1136` the `PlanWaitingHistoryError`, `:1203` the generic
|
|
1508
|
+
* `Error`), so routing the reason into the payload does not necessarily require a
|
|
1509
|
+
* schema change. Log-only on purpose — no behaviour change (see
|
|
1510
|
+
* .agents/notes/20260930-terminal-event-delivery-open-items.md A1).
|
|
1511
|
+
*
|
|
1512
|
+
* `plan.planId` is the row correlation that already exists on
|
|
1513
|
+
* `StatusEventPlan` (see `executeStatusEventPlan`); it is `null` for the
|
|
1514
|
+
* row-less in-memory plans executed without the SQLite store.
|
|
1515
|
+
*/
|
|
1516
|
+
#logDegradedRelease(plan, degradedReason) {
|
|
1517
|
+
console.warn("Herdsman emitted degraded status event", {
|
|
1518
|
+
agentId: plan.agent.id,
|
|
1519
|
+
attempts: plan.attempts ?? 0,
|
|
1520
|
+
degradedReason,
|
|
1521
|
+
herdrSessionName: plan.agent.herdrSessionName,
|
|
1522
|
+
paneId: plan.agent.paneId,
|
|
1523
|
+
planId: plan.planId ?? null,
|
|
1524
|
+
});
|
|
1525
|
+
}
|
|
1013
1526
|
async #appendStatusEvents(input) {
|
|
1014
1527
|
// Invariant: compactHistory is always populated as an object before calling #appendStatusEvents; !input.compactHistory check below is defensive and unreachable in normal operation.
|
|
1015
|
-
if (input.from === input.to || !input.compactHistory)
|
|
1016
|
-
return undefined;
|
|
1017
1528
|
if (this.#stores.agents.isPaneClosed({
|
|
1018
1529
|
herdrSessionName: input.agent.herdrSessionName,
|
|
1019
1530
|
paneId: input.agent.paneId,
|
|
@@ -1042,9 +1553,18 @@ export class AgentIndexService {
|
|
|
1042
1553
|
skipStatusChanged = true;
|
|
1043
1554
|
}
|
|
1044
1555
|
if (hasTerminalAfter && !input.herdrEventKey) {
|
|
1045
|
-
const
|
|
1556
|
+
const isAgyHistory = input.agent.agent === "agy" || input.compactHistory?.source === "antigravity-sqlite";
|
|
1557
|
+
// For an agy/antigravity pane only a terminal event that already
|
|
1558
|
+
// represented a completed turn may count as the delivered baseline: an
|
|
1559
|
+
// `unknown -> idle` startup row would otherwise make the `working -> done`
|
|
1560
|
+
// row of the same round look like a duplicate.
|
|
1561
|
+
const lastTerminal = isAgyHistory
|
|
1562
|
+
? this.#stores.agentEvents.latestCompletedTurnEvent(input.agent.id, input.agent.herdrSessionName)
|
|
1563
|
+
: this.#stores.agentEvents.latestTerminalEvent(input.agent.id, input.agent.herdrSessionName);
|
|
1046
1564
|
if (sameTerminalAssistantContent(input.compactHistory, lastTerminal?.compactHistory, input.agent.agent)) {
|
|
1047
|
-
|
|
1565
|
+
// An agy round whose ref already reached a completed turn is settled: a
|
|
1566
|
+
// retry must skip it instead of exhausting the budget into `discarded`.
|
|
1567
|
+
if ((input.attempts ?? 0) > 0 && !isAgyHistory) {
|
|
1048
1568
|
throw new PlanWaitingHistoryError();
|
|
1049
1569
|
}
|
|
1050
1570
|
console.debug("Herdsman skipping legacy duplicate status transition event with identical history ref", {
|
|
@@ -1092,6 +1612,7 @@ export class AgentIndexService {
|
|
|
1092
1612
|
herdrSessionName: input.agent.herdrSessionName,
|
|
1093
1613
|
paneId: input.agent.paneId,
|
|
1094
1614
|
});
|
|
1615
|
+
this.#throwIfShutdownAbort(controller);
|
|
1095
1616
|
return undefined;
|
|
1096
1617
|
}
|
|
1097
1618
|
if (turn?.received) {
|
|
@@ -1100,7 +1621,61 @@ export class AgentIndexService {
|
|
|
1100
1621
|
forceRefresh: true,
|
|
1101
1622
|
identityChanged: false,
|
|
1102
1623
|
})).snapshot.compactHistory;
|
|
1103
|
-
if (
|
|
1624
|
+
if (this.#stores.agents.isPaneClosed({
|
|
1625
|
+
herdrSessionName: input.agent.herdrSessionName,
|
|
1626
|
+
paneId: input.agent.paneId,
|
|
1627
|
+
paneGeneration: input.agent.paneGeneration ?? null,
|
|
1628
|
+
})) {
|
|
1629
|
+
console.debug("Herdsman skipping status event generation because pane closed after turn signal", {
|
|
1630
|
+
agentId: input.agent.id,
|
|
1631
|
+
herdrSessionName: input.agent.herdrSessionName,
|
|
1632
|
+
paneId: input.agent.paneId,
|
|
1633
|
+
});
|
|
1634
|
+
return undefined;
|
|
1635
|
+
}
|
|
1636
|
+
const freshIsTerminal = isTerminalAssistant(fresh);
|
|
1637
|
+
const expectedText = turn.expectedText?.trim();
|
|
1638
|
+
const lastAssistantText = fresh.lastAssistantMessage?.text?.trim() ?? "";
|
|
1639
|
+
const textMatches = !expectedText || lastAssistantText.endsWith(expectedText);
|
|
1640
|
+
// A client-confirmed turn is a deliverable terminal state: `confirmed`
|
|
1641
|
+
// means Pi verified the final assistant message is on disk, so a
|
|
1642
|
+
// confirmed round is delivered straight through and is never marked
|
|
1643
|
+
// degraded (which is what made #runPlanRow run
|
|
1644
|
+
// invalidateById(..., "degraded_retry") on already-written content).
|
|
1645
|
+
const confirmedTerminal = turn.confirmed === true;
|
|
1646
|
+
const degradeOrRelease = (degradedReason) => {
|
|
1647
|
+
if (confirmedTerminal) {
|
|
1648
|
+
// A confirmed round stays non-degraded (see above), but the reason
|
|
1649
|
+
// is no longer swallowed: this path used to release silently with
|
|
1650
|
+
// an emptied body, so an empty `agent.done` could only be
|
|
1651
|
+
// diagnosed by reading the pane by hand. `degraded: true` must
|
|
1652
|
+
// stay off here — #runPlanRow would `invalidateById(...,
|
|
1653
|
+
// "degraded_retry")` the row it has already written.
|
|
1654
|
+
console.warn("Herdsman released a confirmed pi status event with no deliverable text", {
|
|
1655
|
+
agentId: input.agent.id,
|
|
1656
|
+
degradedReason,
|
|
1657
|
+
herdrSessionName: input.agent.herdrSessionName,
|
|
1658
|
+
paneId: input.agent.paneId,
|
|
1659
|
+
planId: input.planId ?? null,
|
|
1660
|
+
terminalId: input.agent.terminalId,
|
|
1661
|
+
});
|
|
1662
|
+
return { staleSnapshot: false };
|
|
1663
|
+
}
|
|
1664
|
+
this.#logDegradedRelease(input, degradedReason);
|
|
1665
|
+
return { degraded: true, degradedReason, staleSnapshot: false };
|
|
1666
|
+
};
|
|
1667
|
+
if (confirmedTerminal &&
|
|
1668
|
+
freshIsTerminal &&
|
|
1669
|
+
textMatches &&
|
|
1670
|
+
hasNonEmptyAssistantMessage(fresh)) {
|
|
1671
|
+
compactHistory = fresh;
|
|
1672
|
+
}
|
|
1673
|
+
else if (!isAlreadyEmittedInRetry &&
|
|
1674
|
+
freshIsTerminal &&
|
|
1675
|
+
textMatches &&
|
|
1676
|
+
hasNonEmptyAssistantMessage(fresh) &&
|
|
1677
|
+
(isTerminalAssistant(input.compactHistory) ||
|
|
1678
|
+
historyHasAdvanced(fresh, input.compactHistory))) {
|
|
1104
1679
|
compactHistory = fresh;
|
|
1105
1680
|
}
|
|
1106
1681
|
else {
|
|
@@ -1118,7 +1693,54 @@ export class AgentIndexService {
|
|
|
1118
1693
|
!historyHasAdvanced(advanced, baseline, { requireAssistantChange: true })) {
|
|
1119
1694
|
throw new PlanWaitingHistoryError();
|
|
1120
1695
|
}
|
|
1121
|
-
|
|
1696
|
+
// `historyHasAdvanced(..., { requireAssistantChange: true })` compares
|
|
1697
|
+
// the assistant ref/text against the plan baseline. That baseline can
|
|
1698
|
+
// hold this round's answer (a plan row's `compact_history_json` is
|
|
1699
|
+
// written once at plan creation and never updated, and creation
|
|
1700
|
+
// measured 131ms/308ms after the final assistant message reached
|
|
1701
|
+
// disk). It can also hold the answer of a *previous* round — the same
|
|
1702
|
+
// frozen baseline is exactly what the daemon last delivered — so a
|
|
1703
|
+
// round that writes nothing new would otherwise re-release the
|
|
1704
|
+
// previous round's body as if it were this one's. `latestTerminal` is
|
|
1705
|
+
// the latest non-invalidated terminal row read before this round's row
|
|
1706
|
+
// is written: usually the last delivery, possibly an undelivered idle.
|
|
1707
|
+
const staleBaselineDuplicate = sameTerminalAssistantContent(advanced, latestTerminal?.compactHistory, "pi");
|
|
1708
|
+
// A confirmed turn carrying a non-empty terminal assistant message is
|
|
1709
|
+
// trusted as-is, so `no_advance_from_input` stays reserved for rounds
|
|
1710
|
+
// with genuinely no deliverable assistant text (empty or non-terminal
|
|
1711
|
+
// tail).
|
|
1712
|
+
const confirmedDeliverable = confirmedTerminal &&
|
|
1713
|
+
isTerminalAssistant(advanced) &&
|
|
1714
|
+
hasNonEmptyAssistantMessage(advanced) &&
|
|
1715
|
+
!staleBaselineDuplicate;
|
|
1716
|
+
if (!isRetry &&
|
|
1717
|
+
!historyHasAdvanced(advanced, input.compactHistory, { requireAssistantChange: true })) {
|
|
1718
|
+
if (confirmedDeliverable) {
|
|
1719
|
+
// Trust the already-in-place answer: release it as a non-stale
|
|
1720
|
+
// snapshot and without a `degraded` marker.
|
|
1721
|
+
compactHistory = advanced;
|
|
1722
|
+
payloadExtra = { staleSnapshot: false };
|
|
1723
|
+
}
|
|
1724
|
+
else {
|
|
1725
|
+
compactHistory = { ...advanced, lastAssistantMessage: null };
|
|
1726
|
+
payloadExtra = degradeOrRelease(staleBaselineDuplicate ? "stale_baseline_duplicate" : "no_advance_from_input");
|
|
1727
|
+
}
|
|
1728
|
+
}
|
|
1729
|
+
else {
|
|
1730
|
+
const advancedText = advanced.lastAssistantMessage?.text?.trim() ?? "";
|
|
1731
|
+
const advancedMatchesExpected = !expectedText || advancedText.endsWith(expectedText);
|
|
1732
|
+
if (!advancedMatchesExpected) {
|
|
1733
|
+
compactHistory = { ...advanced, lastAssistantMessage: null };
|
|
1734
|
+
payloadExtra = degradeOrRelease("expected_text_mismatch");
|
|
1735
|
+
}
|
|
1736
|
+
else if (!isTerminalAssistant(advanced) || !hasNonEmptyAssistantMessage(advanced)) {
|
|
1737
|
+
compactHistory = { ...advanced, lastAssistantMessage: null };
|
|
1738
|
+
payloadExtra = degradeOrRelease("non_terminal_assistant");
|
|
1739
|
+
}
|
|
1740
|
+
else {
|
|
1741
|
+
compactHistory = advanced;
|
|
1742
|
+
}
|
|
1743
|
+
}
|
|
1122
1744
|
}
|
|
1123
1745
|
console.log(`Herdsman emitted pi agent.${input.to} after turn completion signal (confirmed=${turn.confirmed})`, {
|
|
1124
1746
|
agentId: input.agent.id,
|
|
@@ -1143,7 +1765,21 @@ export class AgentIndexService {
|
|
|
1143
1765
|
if (!isRetry &&
|
|
1144
1766
|
!historyHasAdvanced(advanced, input.compactHistory, { requireAssistantChange: true })) {
|
|
1145
1767
|
compactHistory = { ...advanced, lastAssistantMessage: null };
|
|
1146
|
-
|
|
1768
|
+
this.#logDegradedRelease(input, "no_advance_from_input");
|
|
1769
|
+
payloadExtra = {
|
|
1770
|
+
degraded: true,
|
|
1771
|
+
degradedReason: "no_advance_from_input",
|
|
1772
|
+
staleSnapshot: false,
|
|
1773
|
+
};
|
|
1774
|
+
}
|
|
1775
|
+
else if (!isTerminalAssistant(advanced) || !hasNonEmptyAssistantMessage(advanced)) {
|
|
1776
|
+
compactHistory = { ...advanced, lastAssistantMessage: null };
|
|
1777
|
+
this.#logDegradedRelease(input, "non_terminal_assistant");
|
|
1778
|
+
payloadExtra = {
|
|
1779
|
+
degraded: true,
|
|
1780
|
+
degradedReason: "non_terminal_assistant",
|
|
1781
|
+
staleSnapshot: false,
|
|
1782
|
+
};
|
|
1147
1783
|
}
|
|
1148
1784
|
else {
|
|
1149
1785
|
compactHistory = advanced;
|
|
@@ -1157,15 +1793,21 @@ export class AgentIndexService {
|
|
|
1157
1793
|
}
|
|
1158
1794
|
else if (input.agent.agent !== "pi" && (input.to === "idle" || input.to === "done")) {
|
|
1159
1795
|
// Note: blocked is an interactive intermediate state, exempt from ready gate and empty delivery gate.
|
|
1796
|
+
const isAgyHistory = (compact) => compact?.source === "antigravity-sqlite" || input.agent.agent === "agy";
|
|
1797
|
+
// Only a terminal event that already represented a completed turn may
|
|
1798
|
+
// consume the assistant ref: a late startup idle (`unknown -> idle`),
|
|
1799
|
+
// an `agent.status.changed` row or a discarded plan must not make the
|
|
1800
|
+
// following `working -> done` plan look like a duplicate round.
|
|
1801
|
+
const latestCompletedTurnRef = () => this.#stores.agentEvents.latestCompletedTurnEvent(input.agent.id, input.agent.herdrSessionName)?.compactHistory?.lastAssistantMessage?.ref ?? null;
|
|
1160
1802
|
const isReadyNonPi = (compact) => {
|
|
1161
1803
|
if (!compact || !hasNonEmptyAssistantMessage(compact))
|
|
1162
1804
|
return false;
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
const prevRef = latestTerminal?.compactHistory?.lastAssistantMessage?.ref ?? null;
|
|
1805
|
+
if (isAgyHistory(compact)) {
|
|
1806
|
+
const prevRef = latestCompletedTurnRef();
|
|
1166
1807
|
const currentRef = compact.lastAssistantMessage?.ref ?? null;
|
|
1167
1808
|
return currentRef !== null && currentRef !== prevRef;
|
|
1168
1809
|
}
|
|
1810
|
+
const latestTerminal = this.#stores.agentEvents.latestTerminalEvent(input.agent.id, input.agent.herdrSessionName);
|
|
1169
1811
|
if (!latestTerminal)
|
|
1170
1812
|
return true;
|
|
1171
1813
|
const currentMsg = compact.lastAssistantMessage;
|
|
@@ -1175,11 +1817,22 @@ export class AgentIndexService {
|
|
|
1175
1817
|
const same = sameTerminalAssistantContent(compact, latestTerminal.compactHistory, input.agent.agent);
|
|
1176
1818
|
return !same || refChanged || textChanged;
|
|
1177
1819
|
};
|
|
1820
|
+
// A non-empty assistant ref that was already delivered by a completed
|
|
1821
|
+
// turn is a genuine skip (the round was announced), never a reason to
|
|
1822
|
+
// exhaust the retry budget into `discarded`.
|
|
1823
|
+
const isAlreadyDeliveredCompletedTurn = (compact) => {
|
|
1824
|
+
if (!compact || !isAgyHistory(compact) || !hasNonEmptyAssistantMessage(compact)) {
|
|
1825
|
+
return false;
|
|
1826
|
+
}
|
|
1827
|
+
const currentRef = compact.lastAssistantMessage?.ref ?? null;
|
|
1828
|
+
return currentRef !== null && currentRef === latestCompletedTurnRef();
|
|
1829
|
+
};
|
|
1178
1830
|
const latestTerminalForSkip = this.#stores.agentEvents.latestTerminalEvent(input.agent.id, input.agent.herdrSessionName);
|
|
1179
1831
|
if (input.agent.agent !== "agy" &&
|
|
1180
1832
|
compactHistory?.source !== "antigravity-sqlite" &&
|
|
1181
1833
|
input.herdrEventKey &&
|
|
1182
1834
|
latestTerminalForSkip &&
|
|
1835
|
+
compactHistory &&
|
|
1183
1836
|
hasNonEmptyAssistantMessage(compactHistory)) {
|
|
1184
1837
|
const currentMsg = compactHistory.lastAssistantMessage;
|
|
1185
1838
|
const prevMsg = latestTerminalForSkip.compactHistory?.lastAssistantMessage;
|
|
@@ -1205,6 +1858,7 @@ export class AgentIndexService {
|
|
|
1205
1858
|
herdrSessionName: input.agent.herdrSessionName,
|
|
1206
1859
|
paneId: input.agent.paneId,
|
|
1207
1860
|
});
|
|
1861
|
+
this.#throwIfShutdownAbort(controller);
|
|
1208
1862
|
return undefined;
|
|
1209
1863
|
}
|
|
1210
1864
|
try {
|
|
@@ -1224,6 +1878,16 @@ export class AgentIndexService {
|
|
|
1224
1878
|
}
|
|
1225
1879
|
}
|
|
1226
1880
|
if (!isReadyNonPi(compactHistory)) {
|
|
1881
|
+
if (isAlreadyDeliveredCompletedTurn(compactHistory)) {
|
|
1882
|
+
console.debug("Herdsman skipping non-pi terminal status event because the assistant ref was already delivered by a completed turn", {
|
|
1883
|
+
agent: input.agent.agent,
|
|
1884
|
+
agentId: input.agent.id,
|
|
1885
|
+
from: input.from,
|
|
1886
|
+
ref: compactHistory?.lastAssistantMessage?.ref ?? null,
|
|
1887
|
+
to: input.to,
|
|
1888
|
+
});
|
|
1889
|
+
return undefined;
|
|
1890
|
+
}
|
|
1227
1891
|
throw new PlanWaitingHistoryError();
|
|
1228
1892
|
}
|
|
1229
1893
|
}
|
|
@@ -1239,6 +1903,7 @@ export class AgentIndexService {
|
|
|
1239
1903
|
herdrSessionName: input.agent.herdrSessionName,
|
|
1240
1904
|
paneId: input.agent.paneId,
|
|
1241
1905
|
});
|
|
1906
|
+
this.#throwIfShutdownAbort(controller);
|
|
1242
1907
|
return undefined;
|
|
1243
1908
|
}
|
|
1244
1909
|
const currentAgent = this.#stores.agents.findByPane({
|
|
@@ -1293,7 +1958,7 @@ export class AgentIndexService {
|
|
|
1293
1958
|
? undefined
|
|
1294
1959
|
: this.#appendAndAckSelfEvent({
|
|
1295
1960
|
agentId: targetAgent.id,
|
|
1296
|
-
compactHistory,
|
|
1961
|
+
compactHistory: compactHistory ?? null,
|
|
1297
1962
|
herdrSessionName: targetAgent.herdrSessionName,
|
|
1298
1963
|
idempotencyKey: idempotencyKey("agent.status.changed", targetAgent, input.from, input.to, `${observationId}:${input.herdrEventKey ?? "legacy"}:agent.status.changed`),
|
|
1299
1964
|
paneId: targetAgent.paneId,
|
|
@@ -1307,7 +1972,7 @@ export class AgentIndexService {
|
|
|
1307
1972
|
if (statusType) {
|
|
1308
1973
|
return this.#appendAndAckSelfEvent({
|
|
1309
1974
|
agentId: targetAgent.id,
|
|
1310
|
-
compactHistory,
|
|
1975
|
+
compactHistory: compactHistory ?? null,
|
|
1311
1976
|
herdrSessionName: targetAgent.herdrSessionName,
|
|
1312
1977
|
idempotencyKey: idempotencyKey(statusType, targetAgent, input.from, input.to, `${observationId}:${input.herdrEventKey ?? "legacy"}:${statusType}`),
|
|
1313
1978
|
paneId: targetAgent.paneId,
|
|
@@ -1324,18 +1989,42 @@ export class AgentIndexService {
|
|
|
1324
1989
|
unregister();
|
|
1325
1990
|
}
|
|
1326
1991
|
}
|
|
1327
|
-
|
|
1328
|
-
|
|
1992
|
+
/**
|
|
1993
|
+
* Resolves the metadata a terminal plan outcome is attributed to and whether
|
|
1994
|
+
* the event may keep its `agent_id` foreign key.
|
|
1995
|
+
*
|
|
1996
|
+
* A live `agents` row wins because it carries the current pane generation.
|
|
1997
|
+
* Once the row is gone (pane retired / physically deleted) the plan's own
|
|
1998
|
+
* pane/session/generation are the authority, refined by the newest event of
|
|
1999
|
+
* the same pane + generation that predates the plan, so an old failure is
|
|
2000
|
+
* never stitched onto a newer instance of the same pane. The event is still
|
|
2001
|
+
* written in that case, but as an orphan (`agentRowPresent: false`).
|
|
2002
|
+
*/
|
|
2003
|
+
#resolvePlanOutcomeAgent(agent, planCreatedAt) {
|
|
1329
2004
|
try {
|
|
1330
|
-
agent
|
|
2005
|
+
return { agent: this.#stores.agents.get(agent.id), agentRowPresent: true };
|
|
1331
2006
|
}
|
|
1332
2007
|
catch {
|
|
1333
|
-
const stitched = this.#minimalAgentFromLatestEvent(
|
|
1334
|
-
|
|
1335
|
-
|
|
2008
|
+
const stitched = this.#minimalAgentFromLatestEvent({
|
|
2009
|
+
agentId: agent.id,
|
|
2010
|
+
// The plan row's own creation time is the upper bound: anything newer on
|
|
2011
|
+
// the same pane belongs to a later instance.
|
|
2012
|
+
createdBefore: planCreatedAt ?? Date.now(),
|
|
2013
|
+
herdrSessionName: agent.herdrSessionName,
|
|
2014
|
+
paneGeneration: agent.paneGeneration ?? null,
|
|
2015
|
+
paneId: agent.paneId,
|
|
2016
|
+
});
|
|
2017
|
+
return { agent: stitched ?? agent, agentRowPresent: false };
|
|
1336
2018
|
}
|
|
2019
|
+
}
|
|
2020
|
+
#appendPlanFailedEvent(input) {
|
|
2021
|
+
const originalAgentId = input.agent.id;
|
|
2022
|
+
const resolved = this.#resolvePlanOutcomeAgent(input.agent, input.planCreatedAt);
|
|
2023
|
+
const agent = resolved.agent;
|
|
2024
|
+
const fallbackOutcome = input.fallbackOutcome ??
|
|
2025
|
+
((input.to === "done" || input.to === "blocked") && input.reason === "degraded");
|
|
1337
2026
|
return this.#appendAndAckSelfEvent({
|
|
1338
|
-
agentId: agent.id,
|
|
2027
|
+
agentId: resolved.agentRowPresent ? agent.id : null,
|
|
1339
2028
|
compactHistory: input.compactHistory ?? null,
|
|
1340
2029
|
herdrSessionName: agent.herdrSessionName,
|
|
1341
2030
|
idempotencyKey: `agent.failed:plan:${input.planId}`,
|
|
@@ -1343,7 +2032,11 @@ export class AgentIndexService {
|
|
|
1343
2032
|
paneGeneration: agent.paneGeneration ?? null,
|
|
1344
2033
|
payload: {
|
|
1345
2034
|
agent: agent.agent,
|
|
2035
|
+
// The original id survives in the payload even when the row had to be
|
|
2036
|
+
// written as an orphan, so the failure stays traceable to its agent.
|
|
2037
|
+
agentId: originalAgentId,
|
|
1346
2038
|
attempts: input.attempts,
|
|
2039
|
+
fallbackOutcome,
|
|
1347
2040
|
from: input.from,
|
|
1348
2041
|
herdrSessionName: agent.herdrSessionName,
|
|
1349
2042
|
name: agent.name,
|
|
@@ -1358,6 +2051,36 @@ export class AgentIndexService {
|
|
|
1358
2051
|
workspaceId: agent.workspaceId,
|
|
1359
2052
|
});
|
|
1360
2053
|
}
|
|
2054
|
+
#appendPlanDiscardedEvent(input) {
|
|
2055
|
+
const originalAgentId = input.agent.id;
|
|
2056
|
+
const resolved = this.#resolvePlanOutcomeAgent(input.agent, input.planCreatedAt);
|
|
2057
|
+
const agent = resolved.agent;
|
|
2058
|
+
return this.#appendAndAckSelfEvent({
|
|
2059
|
+
agentId: resolved.agentRowPresent ? agent.id : null,
|
|
2060
|
+
compactHistory: input.compactHistory ?? null,
|
|
2061
|
+
herdrSessionName: agent.herdrSessionName,
|
|
2062
|
+
idempotencyKey: `agent.discarded:plan:${input.planId}`,
|
|
2063
|
+
paneId: agent.paneId,
|
|
2064
|
+
paneGeneration: agent.paneGeneration ?? null,
|
|
2065
|
+
payload: {
|
|
2066
|
+
agent: agent.agent,
|
|
2067
|
+
agentId: originalAgentId,
|
|
2068
|
+
attempts: input.attempts,
|
|
2069
|
+
from: input.from,
|
|
2070
|
+
herdrSessionName: agent.herdrSessionName,
|
|
2071
|
+
name: agent.name,
|
|
2072
|
+
paneId: agent.paneId,
|
|
2073
|
+
planId: input.planId,
|
|
2074
|
+
reason: input.reason ?? "unknown",
|
|
2075
|
+
terminalId: agent.terminalId,
|
|
2076
|
+
to: input.to,
|
|
2077
|
+
workspaceId: agent.workspaceId,
|
|
2078
|
+
},
|
|
2079
|
+
terminalId: agent.terminalId,
|
|
2080
|
+
type: "agent.discarded",
|
|
2081
|
+
workspaceId: agent.workspaceId,
|
|
2082
|
+
});
|
|
2083
|
+
}
|
|
1361
2084
|
#appendAndAckSelfEvent(input) {
|
|
1362
2085
|
const event = this.#stores.agentEvents.append(input);
|
|
1363
2086
|
const owner = this.#stores.agentOrchestratorScopes?.get({
|