@dorokuma/herdsman 0.11.5 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ja.md +9 -1
- package/README.md +21 -1
- package/dist/src/agent-history/pi-reader.d.ts.map +1 -1
- package/dist/src/agent-history/pi-reader.js +10 -2
- package/dist/src/agent-history/pi-reader.js.map +1 -1
- package/dist/src/agent-history/readers.js +6 -1
- package/dist/src/agent-history/readers.js.map +1 -1
- package/dist/src/agent-history/text.d.ts.map +1 -1
- package/dist/src/agent-history/text.js +3 -0
- package/dist/src/agent-history/text.js.map +1 -1
- package/dist/src/config/schema.d.ts +4 -0
- package/dist/src/config/schema.d.ts.map +1 -1
- package/dist/src/config/schema.js +5 -0
- package/dist/src/config/schema.js.map +1 -1
- package/dist/src/daemon/herdr-session-watch-manager.d.ts.map +1 -1
- package/dist/src/daemon/herdr-session-watch-manager.js +1 -0
- package/dist/src/daemon/herdr-session-watch-manager.js.map +1 -1
- package/dist/src/daemon/observability-server.d.ts.map +1 -1
- package/dist/src/daemon/observability-server.js +1 -0
- package/dist/src/daemon/observability-server.js.map +1 -1
- package/dist/src/db/agent-events.d.ts +16 -0
- package/dist/src/db/agent-events.d.ts.map +1 -1
- package/dist/src/db/agent-events.js +27 -1
- package/dist/src/db/agent-events.js.map +1 -1
- package/dist/src/db/status-event-plans.d.ts +35 -3
- package/dist/src/db/status-event-plans.d.ts.map +1 -1
- package/dist/src/db/status-event-plans.js +91 -7
- package/dist/src/db/status-event-plans.js.map +1 -1
- package/dist/src/observability/agent-index-service.d.ts +1 -0
- package/dist/src/observability/agent-index-service.d.ts.map +1 -1
- package/dist/src/observability/agent-index-service.js +423 -64
- package/dist/src/observability/agent-index-service.js.map +1 -1
- package/dist/src/observability/contracts.d.ts +5 -1
- package/dist/src/observability/contracts.d.ts.map +1 -1
- package/dist/src/observability/contracts.js.map +1 -1
- package/dist/src/observability/schemas.d.ts +1 -0
- package/dist/src/observability/schemas.d.ts.map +1 -1
- package/dist/src/observability/schemas.js +1 -0
- package/dist/src/observability/schemas.js.map +1 -1
- package/dist/src/observability/turn-completion.d.ts +3 -0
- package/dist/src/observability/turn-completion.d.ts.map +1 -1
- package/dist/src/observability/turn-completion.js +12 -2
- package/dist/src/observability/turn-completion.js.map +1 -1
- package/package.json +1 -1
|
@@ -22,6 +22,10 @@ export function historyHasAdvanced(current, baseline, options) {
|
|
|
22
22
|
}
|
|
23
23
|
return current.messageCount > (baseline?.messageCount ?? 0);
|
|
24
24
|
}
|
|
25
|
+
function isTerminalAssistant(compact) {
|
|
26
|
+
const stopReason = compact?.lastAssistantMessage?.stopReason;
|
|
27
|
+
return stopReason === "stop" || stopReason === "length";
|
|
28
|
+
}
|
|
25
29
|
export const PLAN_WAITING_HISTORY = "PLAN_WAITING_HISTORY";
|
|
26
30
|
export class PlanWaitingHistoryError extends Error {
|
|
27
31
|
constructor() {
|
|
@@ -50,11 +54,13 @@ export class AgentIndexService {
|
|
|
50
54
|
#sleep;
|
|
51
55
|
#turnCompletions;
|
|
52
56
|
#waitingHistoryTimers = new Map();
|
|
57
|
+
#now;
|
|
53
58
|
constructor(options) {
|
|
54
59
|
this.#clearRetry = options.clearRetry ?? ((timer) => clearTimeout(timer));
|
|
55
60
|
this.#clientFactory = options.clientFactory ?? ((input) => new HerdrSocketClient(input));
|
|
56
61
|
this.#scheduleRetry = options.scheduleRetry ?? ((cb, delay) => setTimeout(cb, delay));
|
|
57
62
|
this.#sleep = options.sleep ?? sleep;
|
|
63
|
+
this.#now = options.now ?? (() => Date.now());
|
|
58
64
|
this.#stores = options.stores;
|
|
59
65
|
this.#turnCompletions = options.turnCompletions;
|
|
60
66
|
if (options.context) {
|
|
@@ -146,8 +152,8 @@ export class AgentIndexService {
|
|
|
146
152
|
if (!store)
|
|
147
153
|
return;
|
|
148
154
|
const current = store.get(planId);
|
|
149
|
-
if (current.status !== "pending"
|
|
150
|
-
this.#
|
|
155
|
+
if (current.status !== "pending") {
|
|
156
|
+
this.#clearWaitingTimer(planId);
|
|
151
157
|
return;
|
|
152
158
|
}
|
|
153
159
|
await this.#enqueueAgentPlan(plan.agent.id, () => this.#retryWaitingPlanRow(planId, plan));
|
|
@@ -167,11 +173,7 @@ export class AgentIndexService {
|
|
|
167
173
|
current = store.get(planId);
|
|
168
174
|
}
|
|
169
175
|
catch {
|
|
170
|
-
this.#
|
|
171
|
-
return;
|
|
172
|
-
}
|
|
173
|
-
if (current.status !== "pending" || current.lastError !== PLAN_WAITING_HISTORY) {
|
|
174
|
-
this.#waitingHistoryTimers.delete(planId);
|
|
176
|
+
this.#clearWaitingTimer(planId);
|
|
175
177
|
return;
|
|
176
178
|
}
|
|
177
179
|
let retryPlan;
|
|
@@ -193,12 +195,36 @@ export class AgentIndexService {
|
|
|
193
195
|
agentId: plan.agent.id,
|
|
194
196
|
error: err,
|
|
195
197
|
});
|
|
196
|
-
store.markRetry(planId, new PlanWaitingHistoryError());
|
|
197
|
-
|
|
198
|
+
const updated = store.markRetry(planId, new PlanWaitingHistoryError());
|
|
199
|
+
if (!updated) {
|
|
200
|
+
this.#clearWaitingTimer(planId);
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
198
203
|
if (updated.status === "pending") {
|
|
199
204
|
this.#scheduleWaitingHistoryRetry(planId, plan);
|
|
200
205
|
}
|
|
206
|
+
else if (updated.status === "discarded") {
|
|
207
|
+
this.#clearWaitingTimer(planId);
|
|
208
|
+
console.info("Herdsman status event plan discarded after max attempts", {
|
|
209
|
+
planId,
|
|
210
|
+
agentId: plan.agent.id,
|
|
211
|
+
attempts: updated.attempts,
|
|
212
|
+
from: plan.from,
|
|
213
|
+
to: plan.to,
|
|
214
|
+
herdrSessionName: plan.agent.herdrSessionName,
|
|
215
|
+
});
|
|
216
|
+
this.#appendPlanDiscardedEvent({
|
|
217
|
+
agent: plan.agent,
|
|
218
|
+
attempts: updated.attempts,
|
|
219
|
+
compactHistory: plan.compactHistory,
|
|
220
|
+
from: plan.from,
|
|
221
|
+
planId,
|
|
222
|
+
reason: updated.lastError,
|
|
223
|
+
to: plan.to,
|
|
224
|
+
});
|
|
225
|
+
}
|
|
201
226
|
else if (updated.status === "failed") {
|
|
227
|
+
this.#clearWaitingTimer(planId);
|
|
202
228
|
console.warn("Herdsman status event plan failed after max attempts", {
|
|
203
229
|
planId,
|
|
204
230
|
agentId: plan.agent.id,
|
|
@@ -281,12 +307,15 @@ export class AgentIndexService {
|
|
|
281
307
|
// rejects: every row is either drained, cancelled, or retried.
|
|
282
308
|
await Promise.allSettled(tasks);
|
|
283
309
|
await this.#backfillFailedPlanEvents();
|
|
310
|
+
await this.#backfillDiscardedPlanEvents();
|
|
284
311
|
}
|
|
285
312
|
async #backfillFailedPlanEvents() {
|
|
286
313
|
const store = this.#stores.statusEventPlans;
|
|
287
314
|
if (!store)
|
|
288
315
|
return;
|
|
289
316
|
for (const row of store.listFailed()) {
|
|
317
|
+
if (row.lastError === PLAN_WAITING_HISTORY)
|
|
318
|
+
continue;
|
|
290
319
|
const existing = this.#stores.sqlite
|
|
291
320
|
.prepare("select 1 from agent_events where herdr_session_name = ? and idempotency_key = ?")
|
|
292
321
|
.get(row.herdrSessionName, `agent.failed:plan:${row.id}`);
|
|
@@ -318,6 +347,42 @@ export class AgentIndexService {
|
|
|
318
347
|
});
|
|
319
348
|
}
|
|
320
349
|
}
|
|
350
|
+
async #backfillDiscardedPlanEvents() {
|
|
351
|
+
const store = this.#stores.statusEventPlans;
|
|
352
|
+
if (!store)
|
|
353
|
+
return;
|
|
354
|
+
for (const row of store.listDiscarded()) {
|
|
355
|
+
const existing = this.#stores.sqlite
|
|
356
|
+
.prepare("select 1 from agent_events where herdr_session_name = ? and idempotency_key = ?")
|
|
357
|
+
.get(row.herdrSessionName, `agent.discarded:plan:${row.id}`);
|
|
358
|
+
if (existing)
|
|
359
|
+
continue;
|
|
360
|
+
let agent;
|
|
361
|
+
try {
|
|
362
|
+
agent = this.#stores.agents.get(row.agentId);
|
|
363
|
+
}
|
|
364
|
+
catch {
|
|
365
|
+
agent = this.#minimalAgentFromLatestEvent(row.agentId, row.herdrSessionName);
|
|
366
|
+
if (!agent) {
|
|
367
|
+
console.warn("Herdsman skipping agent.discarded backfill; no agent row or events", {
|
|
368
|
+
agentId: row.agentId,
|
|
369
|
+
herdrSessionName: row.herdrSessionName,
|
|
370
|
+
planId: row.id,
|
|
371
|
+
});
|
|
372
|
+
continue;
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
this.#appendPlanDiscardedEvent({
|
|
376
|
+
agent,
|
|
377
|
+
attempts: row.attempts,
|
|
378
|
+
compactHistory: row.compactHistory,
|
|
379
|
+
from: row.fromStatus,
|
|
380
|
+
planId: row.id,
|
|
381
|
+
reason: row.lastError,
|
|
382
|
+
to: row.toStatus,
|
|
383
|
+
});
|
|
384
|
+
}
|
|
385
|
+
}
|
|
321
386
|
#minimalAgentFromLatestEvent(agentId, herdrSessionName) {
|
|
322
387
|
const latest = this.#stores.sqlite
|
|
323
388
|
.prepare(`select pane_id, pane_generation, terminal_id, workspace_id, herdr_session_name
|
|
@@ -346,30 +411,48 @@ export class AgentIndexService {
|
|
|
346
411
|
};
|
|
347
412
|
}
|
|
348
413
|
async #drainPlanRow(row) {
|
|
414
|
+
const store = this.#stores.statusEventPlans;
|
|
415
|
+
if (!store)
|
|
416
|
+
return;
|
|
417
|
+
let current;
|
|
418
|
+
try {
|
|
419
|
+
current = store.get(row.id);
|
|
420
|
+
}
|
|
421
|
+
catch {
|
|
422
|
+
this.#clearWaitingTimer(row.id);
|
|
423
|
+
return;
|
|
424
|
+
}
|
|
425
|
+
if (current.status === "completed" ||
|
|
426
|
+
current.status === "cancelled" ||
|
|
427
|
+
current.status === "failed" ||
|
|
428
|
+
current.status === "discarded") {
|
|
429
|
+
this.#clearWaitingTimer(row.id);
|
|
430
|
+
return;
|
|
431
|
+
}
|
|
349
432
|
let agent;
|
|
350
433
|
try {
|
|
351
434
|
agent =
|
|
352
435
|
this.#stores.agents.findByPane({
|
|
353
|
-
herdrSessionName:
|
|
354
|
-
paneId:
|
|
355
|
-
paneGeneration:
|
|
356
|
-
}) ?? this.#stores.agents.get(
|
|
436
|
+
herdrSessionName: current.herdrSessionName,
|
|
437
|
+
paneId: current.paneId,
|
|
438
|
+
paneGeneration: current.paneGeneration,
|
|
439
|
+
}) ?? this.#stores.agents.get(current.agentId);
|
|
357
440
|
}
|
|
358
441
|
catch {
|
|
359
442
|
agent = undefined;
|
|
360
443
|
}
|
|
361
444
|
if (!agent) {
|
|
362
|
-
|
|
445
|
+
store.markCancelled(current.id);
|
|
363
446
|
console.warn("Herdsman cancelling status event plan for missing agent", {
|
|
364
|
-
agentId:
|
|
365
|
-
herdrSessionName:
|
|
366
|
-
paneId:
|
|
367
|
-
from:
|
|
368
|
-
to:
|
|
447
|
+
agentId: current.agentId,
|
|
448
|
+
herdrSessionName: current.herdrSessionName,
|
|
449
|
+
paneId: current.paneId,
|
|
450
|
+
from: current.fromStatus,
|
|
451
|
+
to: current.toStatus,
|
|
369
452
|
});
|
|
370
453
|
return;
|
|
371
454
|
}
|
|
372
|
-
let activeCompact =
|
|
455
|
+
let activeCompact = current.compactHistory;
|
|
373
456
|
try {
|
|
374
457
|
const refreshed = await this.#context.refreshAgent({
|
|
375
458
|
agent,
|
|
@@ -379,52 +462,74 @@ export class AgentIndexService {
|
|
|
379
462
|
activeCompact = refreshed.snapshot.compactHistory;
|
|
380
463
|
}
|
|
381
464
|
catch (err) {
|
|
382
|
-
|
|
383
|
-
|
|
465
|
+
const updated = store.markRetry(current.id, new PlanWaitingHistoryError());
|
|
466
|
+
if (!updated) {
|
|
467
|
+
this.#clearWaitingTimer(current.id);
|
|
468
|
+
return;
|
|
469
|
+
}
|
|
384
470
|
if (updated.status === "pending") {
|
|
385
471
|
console.warn("Herdsman failed to refresh agent during drain plan row, keeping waiting", {
|
|
386
|
-
planId:
|
|
472
|
+
planId: current.id,
|
|
387
473
|
agentId: agent.id,
|
|
388
474
|
error: err,
|
|
389
475
|
});
|
|
390
|
-
this.#scheduleWaitingHistoryRetry(
|
|
476
|
+
this.#scheduleWaitingHistoryRetry(current.id, {
|
|
391
477
|
agent,
|
|
392
|
-
compactHistory:
|
|
393
|
-
from:
|
|
394
|
-
to:
|
|
395
|
-
...(
|
|
478
|
+
compactHistory: current.compactHistory,
|
|
479
|
+
from: current.fromStatus,
|
|
480
|
+
to: current.toStatus,
|
|
481
|
+
...(current.herdrEventKey ? { herdrEventKey: current.herdrEventKey } : {}),
|
|
482
|
+
});
|
|
483
|
+
}
|
|
484
|
+
else if (updated.status === "discarded") {
|
|
485
|
+
this.#clearWaitingTimer(current.id);
|
|
486
|
+
console.info("Herdsman plan marked discarded during drain", {
|
|
487
|
+
planId: current.id,
|
|
488
|
+
agentId: agent.id,
|
|
489
|
+
from: current.fromStatus,
|
|
490
|
+
to: current.toStatus,
|
|
491
|
+
reason: updated.lastError,
|
|
492
|
+
});
|
|
493
|
+
this.#appendPlanDiscardedEvent({
|
|
494
|
+
agent,
|
|
495
|
+
attempts: updated.attempts,
|
|
496
|
+
compactHistory: current.compactHistory,
|
|
497
|
+
from: current.fromStatus,
|
|
498
|
+
planId: current.id,
|
|
499
|
+
reason: updated.lastError,
|
|
500
|
+
to: current.toStatus,
|
|
396
501
|
});
|
|
397
502
|
}
|
|
398
503
|
else if (updated.status === "failed") {
|
|
399
|
-
this.#clearWaitingTimer(
|
|
504
|
+
this.#clearWaitingTimer(current.id);
|
|
400
505
|
console.warn("Herdsman plan marked failed", {
|
|
401
|
-
planId:
|
|
506
|
+
planId: current.id,
|
|
402
507
|
agentId: agent.id,
|
|
403
|
-
from:
|
|
404
|
-
to:
|
|
508
|
+
from: current.fromStatus,
|
|
509
|
+
to: current.toStatus,
|
|
405
510
|
reason: updated.lastError,
|
|
406
511
|
});
|
|
407
512
|
this.#appendPlanFailedEvent({
|
|
408
513
|
agent,
|
|
409
514
|
attempts: updated.attempts,
|
|
410
|
-
compactHistory:
|
|
411
|
-
from:
|
|
412
|
-
planId:
|
|
515
|
+
compactHistory: current.compactHistory,
|
|
516
|
+
from: current.fromStatus,
|
|
517
|
+
planId: current.id,
|
|
413
518
|
reason: updated.lastError,
|
|
414
|
-
to:
|
|
519
|
+
to: current.toStatus,
|
|
415
520
|
});
|
|
416
521
|
}
|
|
417
522
|
return;
|
|
418
523
|
}
|
|
419
524
|
const plan = {
|
|
420
525
|
agent,
|
|
421
|
-
attempts:
|
|
526
|
+
attempts: current.attempts,
|
|
422
527
|
compactHistory: activeCompact,
|
|
423
|
-
from:
|
|
424
|
-
to:
|
|
425
|
-
...(
|
|
528
|
+
from: current.fromStatus,
|
|
529
|
+
to: current.toStatus,
|
|
530
|
+
...(current.herdrEventKey ? { herdrEventKey: current.herdrEventKey } : {}),
|
|
426
531
|
};
|
|
427
|
-
await this.#runPlanRow(
|
|
532
|
+
await this.#runPlanRow(current, plan);
|
|
428
533
|
}
|
|
429
534
|
#enqueueAgentPlan(agentId, operation) {
|
|
430
535
|
const previous = this.#planTailByAgent.get(agentId) ?? Promise.resolve();
|
|
@@ -445,7 +550,8 @@ export class AgentIndexService {
|
|
|
445
550
|
const current = store.get(row.id);
|
|
446
551
|
if (current.status === "completed" ||
|
|
447
552
|
current.status === "cancelled" ||
|
|
448
|
-
current.status === "failed"
|
|
553
|
+
current.status === "failed" ||
|
|
554
|
+
current.status === "discarded") {
|
|
449
555
|
this.#clearWaitingTimer(row.id);
|
|
450
556
|
return undefined;
|
|
451
557
|
}
|
|
@@ -473,13 +579,35 @@ export class AgentIndexService {
|
|
|
473
579
|
agentId: plan.agent.id,
|
|
474
580
|
error: err,
|
|
475
581
|
});
|
|
476
|
-
store.markRetry(row.id, new PlanWaitingHistoryError());
|
|
477
|
-
|
|
582
|
+
const updated = store.markRetry(row.id, new PlanWaitingHistoryError());
|
|
583
|
+
if (!updated) {
|
|
584
|
+
this.#clearWaitingTimer(row.id);
|
|
585
|
+
return undefined;
|
|
586
|
+
}
|
|
478
587
|
if (updated.status === "pending") {
|
|
479
588
|
this.#scheduleWaitingHistoryRetry(row.id, plan);
|
|
480
589
|
}
|
|
481
590
|
else {
|
|
482
591
|
this.#clearWaitingTimer(row.id);
|
|
592
|
+
if (updated.status === "discarded") {
|
|
593
|
+
console.info("Herdsman status event plan discarded after max attempts", {
|
|
594
|
+
planId: row.id,
|
|
595
|
+
agentId: row.agentId,
|
|
596
|
+
attempts: updated.attempts,
|
|
597
|
+
from: row.fromStatus,
|
|
598
|
+
to: row.toStatus,
|
|
599
|
+
herdrSessionName: row.herdrSessionName,
|
|
600
|
+
});
|
|
601
|
+
return this.#appendPlanDiscardedEvent({
|
|
602
|
+
agent: plan.agent,
|
|
603
|
+
attempts: updated.attempts,
|
|
604
|
+
compactHistory: plan.compactHistory,
|
|
605
|
+
from: row.fromStatus,
|
|
606
|
+
planId: row.id,
|
|
607
|
+
reason: updated.lastError,
|
|
608
|
+
to: row.toStatus,
|
|
609
|
+
});
|
|
610
|
+
}
|
|
483
611
|
if (updated.status === "failed") {
|
|
484
612
|
console.warn("Herdsman status event plan failed after max attempts", {
|
|
485
613
|
planId: row.id,
|
|
@@ -521,18 +649,80 @@ export class AgentIndexService {
|
|
|
521
649
|
return undefined;
|
|
522
650
|
}
|
|
523
651
|
this.#clearWaitingTimer(row.id);
|
|
652
|
+
const eventPayload = event?.payload;
|
|
653
|
+
const isDegraded = !!event &&
|
|
654
|
+
typeof eventPayload === "object" &&
|
|
655
|
+
eventPayload !== null &&
|
|
656
|
+
eventPayload.degraded === true;
|
|
657
|
+
if (isDegraded) {
|
|
658
|
+
const updated = store.markRetry(row.id, new Error("degraded"));
|
|
659
|
+
if (!updated) {
|
|
660
|
+
return event;
|
|
661
|
+
}
|
|
662
|
+
if (updated.status === "pending") {
|
|
663
|
+
this.#scheduleWaitingHistoryRetry(row.id, plan);
|
|
664
|
+
}
|
|
665
|
+
else {
|
|
666
|
+
this.#clearWaitingTimer(row.id);
|
|
667
|
+
if (updated.status === "discarded") {
|
|
668
|
+
return this.#appendPlanDiscardedEvent({
|
|
669
|
+
agent: plan.agent,
|
|
670
|
+
attempts: updated.attempts,
|
|
671
|
+
compactHistory: activePlan.compactHistory,
|
|
672
|
+
from: row.fromStatus,
|
|
673
|
+
planId: row.id,
|
|
674
|
+
reason: updated.lastError,
|
|
675
|
+
to: row.toStatus,
|
|
676
|
+
});
|
|
677
|
+
}
|
|
678
|
+
if (updated.status === "failed") {
|
|
679
|
+
return this.#appendPlanFailedEvent({
|
|
680
|
+
agent: plan.agent,
|
|
681
|
+
attempts: updated.attempts,
|
|
682
|
+
compactHistory: activePlan.compactHistory,
|
|
683
|
+
from: row.fromStatus,
|
|
684
|
+
planId: row.id,
|
|
685
|
+
reason: updated.lastError,
|
|
686
|
+
to: row.toStatus,
|
|
687
|
+
});
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
return event;
|
|
691
|
+
}
|
|
524
692
|
store.markCompleted(row.id);
|
|
525
693
|
return event;
|
|
526
694
|
}
|
|
527
695
|
catch (error) {
|
|
528
696
|
if (error instanceof PlanWaitingHistoryError) {
|
|
529
|
-
store.markRetry(row.id, error);
|
|
530
|
-
|
|
697
|
+
const updated = store.markRetry(row.id, error);
|
|
698
|
+
if (!updated) {
|
|
699
|
+
this.#clearWaitingTimer(row.id);
|
|
700
|
+
return undefined;
|
|
701
|
+
}
|
|
531
702
|
if (updated.status === "pending") {
|
|
532
703
|
this.#scheduleWaitingHistoryRetry(row.id, plan);
|
|
533
704
|
}
|
|
534
705
|
else {
|
|
535
706
|
this.#clearWaitingTimer(row.id);
|
|
707
|
+
if (updated.status === "discarded") {
|
|
708
|
+
console.info("Herdsman status event plan discarded after max attempts", {
|
|
709
|
+
planId: row.id,
|
|
710
|
+
agentId: row.agentId,
|
|
711
|
+
attempts: updated.attempts,
|
|
712
|
+
from: row.fromStatus,
|
|
713
|
+
to: row.toStatus,
|
|
714
|
+
herdrSessionName: row.herdrSessionName,
|
|
715
|
+
});
|
|
716
|
+
return this.#appendPlanDiscardedEvent({
|
|
717
|
+
agent: plan.agent,
|
|
718
|
+
attempts: updated.attempts,
|
|
719
|
+
compactHistory: activePlan.compactHistory,
|
|
720
|
+
from: row.fromStatus,
|
|
721
|
+
planId: row.id,
|
|
722
|
+
reason: updated.lastError,
|
|
723
|
+
to: row.toStatus,
|
|
724
|
+
});
|
|
725
|
+
}
|
|
536
726
|
if (updated.status === "failed") {
|
|
537
727
|
console.warn("Herdsman status event plan failed after max attempts", {
|
|
538
728
|
planId: row.id,
|
|
@@ -569,8 +759,29 @@ export class AgentIndexService {
|
|
|
569
759
|
return undefined;
|
|
570
760
|
}
|
|
571
761
|
this.#clearWaitingTimer(row.id);
|
|
572
|
-
store.markRetry(row.id, error);
|
|
573
|
-
|
|
762
|
+
const updated = store.markRetry(row.id, error);
|
|
763
|
+
if (!updated) {
|
|
764
|
+
return undefined;
|
|
765
|
+
}
|
|
766
|
+
if (updated.status === "discarded") {
|
|
767
|
+
console.info("Herdsman status event plan discarded after max attempts", {
|
|
768
|
+
planId: row.id,
|
|
769
|
+
agentId: row.agentId,
|
|
770
|
+
attempts: updated.attempts,
|
|
771
|
+
from: row.fromStatus,
|
|
772
|
+
to: row.toStatus,
|
|
773
|
+
herdrSessionName: row.herdrSessionName,
|
|
774
|
+
});
|
|
775
|
+
return this.#appendPlanDiscardedEvent({
|
|
776
|
+
agent: plan.agent,
|
|
777
|
+
attempts: updated.attempts,
|
|
778
|
+
compactHistory: activePlan.compactHistory,
|
|
779
|
+
from: row.fromStatus,
|
|
780
|
+
planId: row.id,
|
|
781
|
+
reason: updated.lastError,
|
|
782
|
+
to: row.toStatus,
|
|
783
|
+
});
|
|
784
|
+
}
|
|
574
785
|
if (updated.status === "failed") {
|
|
575
786
|
console.warn("Herdsman status event plan failed after max attempts", {
|
|
576
787
|
planId: row.id,
|
|
@@ -595,6 +806,10 @@ export class AgentIndexService {
|
|
|
595
806
|
}
|
|
596
807
|
async #waitForHistoryAdvance(input) {
|
|
597
808
|
const maxAttempts = input.maxAttempts ?? 8;
|
|
809
|
+
const baseDelay = 500;
|
|
810
|
+
const maxDelay = 16000;
|
|
811
|
+
const maxTotalMs = 30000;
|
|
812
|
+
let delay = baseDelay;
|
|
598
813
|
let refreshed = input.initial
|
|
599
814
|
? { snapshot: { compactHistory: input.initial } }
|
|
600
815
|
: await this.#context.refreshAgent({
|
|
@@ -602,20 +817,28 @@ export class AgentIndexService {
|
|
|
602
817
|
forceRefresh: true,
|
|
603
818
|
identityChanged: false,
|
|
604
819
|
});
|
|
820
|
+
const start = this.#now();
|
|
605
821
|
for (let attempt = 0; attempt < maxAttempts &&
|
|
606
822
|
!historyHasAdvanced(refreshed.snapshot.compactHistory, input.baseline, {
|
|
607
823
|
requireAssistantChange: input.requireAssistantChange,
|
|
608
824
|
}) &&
|
|
609
|
-
!input.controller.signal.aborted
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
825
|
+
!input.controller.signal.aborted &&
|
|
826
|
+
this.#now() - start < maxTotalMs; attempt += 1) {
|
|
827
|
+
await this.#sleep(delay, input.controller.signal);
|
|
828
|
+
if (input.controller.signal.aborted) {
|
|
829
|
+
console.debug("Herdsman retrying status event plan because wait was aborted during history refresh", {
|
|
830
|
+
agentId: input.agent.id,
|
|
613
831
|
herdrSessionName: input.agent.herdrSessionName,
|
|
614
832
|
paneId: input.agent.paneId,
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
833
|
+
});
|
|
834
|
+
throw new PlanWaitingHistoryError();
|
|
835
|
+
}
|
|
836
|
+
if (this.#stores.agents.isPaneClosed({
|
|
837
|
+
herdrSessionName: input.agent.herdrSessionName,
|
|
838
|
+
paneId: input.agent.paneId,
|
|
839
|
+
paneGeneration: input.agent.paneGeneration ?? null,
|
|
840
|
+
})) {
|
|
841
|
+
console.debug("Herdsman skipping status event generation because pane closed during history refresh", {
|
|
619
842
|
agentId: input.agent.id,
|
|
620
843
|
herdrSessionName: input.agent.herdrSessionName,
|
|
621
844
|
paneId: input.agent.paneId,
|
|
@@ -627,6 +850,7 @@ export class AgentIndexService {
|
|
|
627
850
|
forceRefresh: true,
|
|
628
851
|
identityChanged: false,
|
|
629
852
|
});
|
|
853
|
+
delay = Math.min(delay * 2, maxDelay);
|
|
630
854
|
}
|
|
631
855
|
return refreshed.snapshot.compactHistory;
|
|
632
856
|
}
|
|
@@ -1042,9 +1266,18 @@ export class AgentIndexService {
|
|
|
1042
1266
|
skipStatusChanged = true;
|
|
1043
1267
|
}
|
|
1044
1268
|
if (hasTerminalAfter && !input.herdrEventKey) {
|
|
1045
|
-
const
|
|
1269
|
+
const isAgyHistory = input.agent.agent === "agy" || input.compactHistory.source === "antigravity-sqlite";
|
|
1270
|
+
// For an agy/antigravity pane only a terminal event that already
|
|
1271
|
+
// represented a completed turn may count as the delivered baseline: an
|
|
1272
|
+
// `unknown -> idle` startup row would otherwise make the `working -> done`
|
|
1273
|
+
// row of the same round look like a duplicate.
|
|
1274
|
+
const lastTerminal = isAgyHistory
|
|
1275
|
+
? this.#stores.agentEvents.latestCompletedTurnEvent(input.agent.id, input.agent.herdrSessionName)
|
|
1276
|
+
: this.#stores.agentEvents.latestTerminalEvent(input.agent.id, input.agent.herdrSessionName);
|
|
1046
1277
|
if (sameTerminalAssistantContent(input.compactHistory, lastTerminal?.compactHistory, input.agent.agent)) {
|
|
1047
|
-
|
|
1278
|
+
// An agy round whose ref already reached a completed turn is settled: a
|
|
1279
|
+
// retry must skip it instead of exhausting the budget into `discarded`.
|
|
1280
|
+
if ((input.attempts ?? 0) > 0 && !isAgyHistory) {
|
|
1048
1281
|
throw new PlanWaitingHistoryError();
|
|
1049
1282
|
}
|
|
1050
1283
|
console.debug("Herdsman skipping legacy duplicate status transition event with identical history ref", {
|
|
@@ -1100,7 +1333,28 @@ export class AgentIndexService {
|
|
|
1100
1333
|
forceRefresh: true,
|
|
1101
1334
|
identityChanged: false,
|
|
1102
1335
|
})).snapshot.compactHistory;
|
|
1103
|
-
if (
|
|
1336
|
+
if (this.#stores.agents.isPaneClosed({
|
|
1337
|
+
herdrSessionName: input.agent.herdrSessionName,
|
|
1338
|
+
paneId: input.agent.paneId,
|
|
1339
|
+
paneGeneration: input.agent.paneGeneration ?? null,
|
|
1340
|
+
})) {
|
|
1341
|
+
console.debug("Herdsman skipping status event generation because pane closed after turn signal", {
|
|
1342
|
+
agentId: input.agent.id,
|
|
1343
|
+
herdrSessionName: input.agent.herdrSessionName,
|
|
1344
|
+
paneId: input.agent.paneId,
|
|
1345
|
+
});
|
|
1346
|
+
return undefined;
|
|
1347
|
+
}
|
|
1348
|
+
const freshIsTerminal = isTerminalAssistant(fresh);
|
|
1349
|
+
const expectedText = turn.expectedText?.trim();
|
|
1350
|
+
const lastAssistantText = fresh.lastAssistantMessage?.text?.trim() ?? "";
|
|
1351
|
+
const textMatches = !expectedText || lastAssistantText.endsWith(expectedText);
|
|
1352
|
+
if (!isAlreadyEmittedInRetry &&
|
|
1353
|
+
freshIsTerminal &&
|
|
1354
|
+
textMatches &&
|
|
1355
|
+
hasNonEmptyAssistantMessage(fresh) &&
|
|
1356
|
+
(isTerminalAssistant(input.compactHistory) ||
|
|
1357
|
+
historyHasAdvanced(fresh, input.compactHistory))) {
|
|
1104
1358
|
compactHistory = fresh;
|
|
1105
1359
|
}
|
|
1106
1360
|
else {
|
|
@@ -1118,7 +1372,38 @@ export class AgentIndexService {
|
|
|
1118
1372
|
!historyHasAdvanced(advanced, baseline, { requireAssistantChange: true })) {
|
|
1119
1373
|
throw new PlanWaitingHistoryError();
|
|
1120
1374
|
}
|
|
1121
|
-
|
|
1375
|
+
if (!isRetry &&
|
|
1376
|
+
!historyHasAdvanced(advanced, input.compactHistory, { requireAssistantChange: true })) {
|
|
1377
|
+
compactHistory = { ...advanced, lastAssistantMessage: null };
|
|
1378
|
+
payloadExtra = {
|
|
1379
|
+
degraded: true,
|
|
1380
|
+
degradedReason: "no_advance_from_input",
|
|
1381
|
+
staleSnapshot: false,
|
|
1382
|
+
};
|
|
1383
|
+
}
|
|
1384
|
+
else {
|
|
1385
|
+
const advancedText = advanced.lastAssistantMessage?.text?.trim() ?? "";
|
|
1386
|
+
const advancedMatchesExpected = !expectedText || advancedText.endsWith(expectedText);
|
|
1387
|
+
if (!advancedMatchesExpected) {
|
|
1388
|
+
compactHistory = { ...advanced, lastAssistantMessage: null };
|
|
1389
|
+
payloadExtra = {
|
|
1390
|
+
degraded: true,
|
|
1391
|
+
degradedReason: "expected_text_mismatch",
|
|
1392
|
+
staleSnapshot: false,
|
|
1393
|
+
};
|
|
1394
|
+
}
|
|
1395
|
+
else if (!isTerminalAssistant(advanced)) {
|
|
1396
|
+
compactHistory = { ...advanced, lastAssistantMessage: null };
|
|
1397
|
+
payloadExtra = {
|
|
1398
|
+
degraded: true,
|
|
1399
|
+
degradedReason: "non_terminal_assistant",
|
|
1400
|
+
staleSnapshot: false,
|
|
1401
|
+
};
|
|
1402
|
+
}
|
|
1403
|
+
else {
|
|
1404
|
+
compactHistory = advanced;
|
|
1405
|
+
}
|
|
1406
|
+
}
|
|
1122
1407
|
}
|
|
1123
1408
|
console.log(`Herdsman emitted pi agent.${input.to} after turn completion signal (confirmed=${turn.confirmed})`, {
|
|
1124
1409
|
agentId: input.agent.id,
|
|
@@ -1143,7 +1428,19 @@ export class AgentIndexService {
|
|
|
1143
1428
|
if (!isRetry &&
|
|
1144
1429
|
!historyHasAdvanced(advanced, input.compactHistory, { requireAssistantChange: true })) {
|
|
1145
1430
|
compactHistory = { ...advanced, lastAssistantMessage: null };
|
|
1146
|
-
payloadExtra = {
|
|
1431
|
+
payloadExtra = {
|
|
1432
|
+
degraded: true,
|
|
1433
|
+
degradedReason: "no_advance_from_input",
|
|
1434
|
+
staleSnapshot: false,
|
|
1435
|
+
};
|
|
1436
|
+
}
|
|
1437
|
+
else if (!isTerminalAssistant(advanced)) {
|
|
1438
|
+
compactHistory = { ...advanced, lastAssistantMessage: null };
|
|
1439
|
+
payloadExtra = {
|
|
1440
|
+
degraded: true,
|
|
1441
|
+
degradedReason: "non_terminal_assistant",
|
|
1442
|
+
staleSnapshot: false,
|
|
1443
|
+
};
|
|
1147
1444
|
}
|
|
1148
1445
|
else {
|
|
1149
1446
|
compactHistory = advanced;
|
|
@@ -1157,15 +1454,21 @@ export class AgentIndexService {
|
|
|
1157
1454
|
}
|
|
1158
1455
|
else if (input.agent.agent !== "pi" && (input.to === "idle" || input.to === "done")) {
|
|
1159
1456
|
// Note: blocked is an interactive intermediate state, exempt from ready gate and empty delivery gate.
|
|
1457
|
+
const isAgyHistory = (compact) => compact?.source === "antigravity-sqlite" || input.agent.agent === "agy";
|
|
1458
|
+
// Only a terminal event that already represented a completed turn may
|
|
1459
|
+
// consume the assistant ref: a late startup idle (`unknown -> idle`),
|
|
1460
|
+
// an `agent.status.changed` row or a discarded plan must not make the
|
|
1461
|
+
// following `working -> done` plan look like a duplicate round.
|
|
1462
|
+
const latestCompletedTurnRef = () => this.#stores.agentEvents.latestCompletedTurnEvent(input.agent.id, input.agent.herdrSessionName)?.compactHistory?.lastAssistantMessage?.ref ?? null;
|
|
1160
1463
|
const isReadyNonPi = (compact) => {
|
|
1161
1464
|
if (!compact || !hasNonEmptyAssistantMessage(compact))
|
|
1162
1465
|
return false;
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
const prevRef = latestTerminal?.compactHistory?.lastAssistantMessage?.ref ?? null;
|
|
1466
|
+
if (isAgyHistory(compact)) {
|
|
1467
|
+
const prevRef = latestCompletedTurnRef();
|
|
1166
1468
|
const currentRef = compact.lastAssistantMessage?.ref ?? null;
|
|
1167
1469
|
return currentRef !== null && currentRef !== prevRef;
|
|
1168
1470
|
}
|
|
1471
|
+
const latestTerminal = this.#stores.agentEvents.latestTerminalEvent(input.agent.id, input.agent.herdrSessionName);
|
|
1169
1472
|
if (!latestTerminal)
|
|
1170
1473
|
return true;
|
|
1171
1474
|
const currentMsg = compact.lastAssistantMessage;
|
|
@@ -1175,6 +1478,16 @@ export class AgentIndexService {
|
|
|
1175
1478
|
const same = sameTerminalAssistantContent(compact, latestTerminal.compactHistory, input.agent.agent);
|
|
1176
1479
|
return !same || refChanged || textChanged;
|
|
1177
1480
|
};
|
|
1481
|
+
// A non-empty assistant ref that was already delivered by a completed
|
|
1482
|
+
// turn is a genuine skip (the round was announced), never a reason to
|
|
1483
|
+
// exhaust the retry budget into `discarded`.
|
|
1484
|
+
const isAlreadyDeliveredCompletedTurn = (compact) => {
|
|
1485
|
+
if (!compact || !isAgyHistory(compact) || !hasNonEmptyAssistantMessage(compact)) {
|
|
1486
|
+
return false;
|
|
1487
|
+
}
|
|
1488
|
+
const currentRef = compact.lastAssistantMessage?.ref ?? null;
|
|
1489
|
+
return currentRef !== null && currentRef === latestCompletedTurnRef();
|
|
1490
|
+
};
|
|
1178
1491
|
const latestTerminalForSkip = this.#stores.agentEvents.latestTerminalEvent(input.agent.id, input.agent.herdrSessionName);
|
|
1179
1492
|
if (input.agent.agent !== "agy" &&
|
|
1180
1493
|
compactHistory?.source !== "antigravity-sqlite" &&
|
|
@@ -1224,6 +1537,16 @@ export class AgentIndexService {
|
|
|
1224
1537
|
}
|
|
1225
1538
|
}
|
|
1226
1539
|
if (!isReadyNonPi(compactHistory)) {
|
|
1540
|
+
if (isAlreadyDeliveredCompletedTurn(compactHistory)) {
|
|
1541
|
+
console.debug("Herdsman skipping non-pi terminal status event because the assistant ref was already delivered by a completed turn", {
|
|
1542
|
+
agent: input.agent.agent,
|
|
1543
|
+
agentId: input.agent.id,
|
|
1544
|
+
from: input.from,
|
|
1545
|
+
ref: compactHistory?.lastAssistantMessage?.ref ?? null,
|
|
1546
|
+
to: input.to,
|
|
1547
|
+
});
|
|
1548
|
+
return undefined;
|
|
1549
|
+
}
|
|
1227
1550
|
throw new PlanWaitingHistoryError();
|
|
1228
1551
|
}
|
|
1229
1552
|
}
|
|
@@ -1358,6 +1681,42 @@ export class AgentIndexService {
|
|
|
1358
1681
|
workspaceId: agent.workspaceId,
|
|
1359
1682
|
});
|
|
1360
1683
|
}
|
|
1684
|
+
#appendPlanDiscardedEvent(input) {
|
|
1685
|
+
let agent = input.agent;
|
|
1686
|
+
try {
|
|
1687
|
+
agent = this.#stores.agents.get(input.agent.id);
|
|
1688
|
+
}
|
|
1689
|
+
catch {
|
|
1690
|
+
const stitched = this.#minimalAgentFromLatestEvent(input.agent.id, input.agent.herdrSessionName);
|
|
1691
|
+
if (stitched)
|
|
1692
|
+
agent = stitched;
|
|
1693
|
+
}
|
|
1694
|
+
return this.#appendAndAckSelfEvent({
|
|
1695
|
+
agentId: agent.id,
|
|
1696
|
+
compactHistory: input.compactHistory ?? null,
|
|
1697
|
+
herdrSessionName: agent.herdrSessionName,
|
|
1698
|
+
idempotencyKey: `agent.discarded:plan:${input.planId}`,
|
|
1699
|
+
paneId: agent.paneId,
|
|
1700
|
+
paneGeneration: agent.paneGeneration ?? null,
|
|
1701
|
+
payload: {
|
|
1702
|
+
agent: agent.agent,
|
|
1703
|
+
agentId: agent.id,
|
|
1704
|
+
attempts: input.attempts,
|
|
1705
|
+
from: input.from,
|
|
1706
|
+
herdrSessionName: agent.herdrSessionName,
|
|
1707
|
+
name: agent.name,
|
|
1708
|
+
paneId: agent.paneId,
|
|
1709
|
+
planId: input.planId,
|
|
1710
|
+
reason: input.reason ?? "unknown",
|
|
1711
|
+
terminalId: agent.terminalId,
|
|
1712
|
+
to: input.to,
|
|
1713
|
+
workspaceId: agent.workspaceId,
|
|
1714
|
+
},
|
|
1715
|
+
terminalId: agent.terminalId,
|
|
1716
|
+
type: "agent.discarded",
|
|
1717
|
+
workspaceId: agent.workspaceId,
|
|
1718
|
+
});
|
|
1719
|
+
}
|
|
1361
1720
|
#appendAndAckSelfEvent(input) {
|
|
1362
1721
|
const event = this.#stores.agentEvents.append(input);
|
|
1363
1722
|
const owner = this.#stores.agentOrchestratorScopes?.get({
|