@animalabs/agent-framework 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -1
- package/dist/src/agent.d.ts +32 -0
- package/dist/src/agent.d.ts.map +1 -1
- package/dist/src/agent.js +55 -0
- package/dist/src/agent.js.map +1 -1
- package/dist/src/api/server.d.ts +9 -0
- package/dist/src/api/server.d.ts.map +1 -1
- package/dist/src/api/server.js +156 -0
- package/dist/src/api/server.js.map +1 -1
- package/dist/src/api/types.d.ts +15 -0
- package/dist/src/api/types.d.ts.map +1 -1
- package/dist/src/framework.d.ts +423 -3
- package/dist/src/framework.d.ts.map +1 -1
- package/dist/src/framework.js +1857 -75
- package/dist/src/framework.js.map +1 -1
- package/dist/src/gate/event-gate.d.ts +16 -0
- package/dist/src/gate/event-gate.d.ts.map +1 -1
- package/dist/src/gate/event-gate.js +41 -3
- package/dist/src/gate/event-gate.js.map +1 -1
- package/dist/src/index.d.ts +4 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +3 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/mcpl/channel-registry.d.ts +172 -0
- package/dist/src/mcpl/channel-registry.d.ts.map +1 -1
- package/dist/src/mcpl/channel-registry.js +444 -2
- package/dist/src/mcpl/channel-registry.js.map +1 -1
- package/dist/src/mcpl/server-connection.d.ts +9 -0
- package/dist/src/mcpl/server-connection.d.ts.map +1 -1
- package/dist/src/mcpl/server-connection.js +71 -13
- package/dist/src/mcpl/server-connection.js.map +1 -1
- package/dist/src/modules/history/index.d.ts +89 -13
- package/dist/src/modules/history/index.d.ts.map +1 -1
- package/dist/src/modules/history/index.js +655 -24
- package/dist/src/modules/history/index.js.map +1 -1
- package/dist/src/operator-log.d.ts +78 -0
- package/dist/src/operator-log.d.ts.map +1 -0
- package/dist/src/operator-log.js +102 -0
- package/dist/src/operator-log.js.map +1 -0
- package/dist/src/recovery/discord-awareness-outbox.d.ts +10 -0
- package/dist/src/recovery/discord-awareness-outbox.d.ts.map +1 -1
- package/dist/src/recovery/discord-awareness-outbox.js +20 -0
- package/dist/src/recovery/discord-awareness-outbox.js.map +1 -1
- package/dist/src/types/framework.d.ts +56 -0
- package/dist/src/types/framework.d.ts.map +1 -1
- package/dist/src/types/trace.d.ts +36 -0
- package/dist/src/types/trace.d.ts.map +1 -1
- package/dist/test/agent-runtime-settings.test.js +199 -1
- package/dist/test/agent-runtime-settings.test.js.map +1 -1
- package/dist/test/api-admin-gate.test.d.ts +2 -0
- package/dist/test/api-admin-gate.test.d.ts.map +1 -0
- package/dist/test/api-admin-gate.test.js +116 -0
- package/dist/test/api-admin-gate.test.js.map +1 -0
- package/dist/test/channel-label-history.test.d.ts +2 -0
- package/dist/test/channel-label-history.test.d.ts.map +1 -0
- package/dist/test/channel-label-history.test.js +424 -0
- package/dist/test/channel-label-history.test.js.map +1 -0
- package/dist/test/discord-awareness-outbox.test.js +16 -0
- package/dist/test/discord-awareness-outbox.test.js.map +1 -1
- package/dist/test/event-gate.test.js +44 -0
- package/dist/test/event-gate.test.js.map +1 -1
- package/dist/test/helpers/quiesce-crash-child.d.ts +2 -0
- package/dist/test/helpers/quiesce-crash-child.d.ts.map +1 -0
- package/dist/test/helpers/quiesce-crash-child.js +156 -0
- package/dist/test/helpers/quiesce-crash-child.js.map +1 -0
- package/dist/test/history-module-overview.test.d.ts +2 -0
- package/dist/test/history-module-overview.test.d.ts.map +1 -0
- package/dist/test/history-module-overview.test.js +711 -0
- package/dist/test/history-module-overview.test.js.map +1 -0
- package/dist/test/history-module.test.js +146 -0
- package/dist/test/history-module.test.js.map +1 -1
- package/dist/test/host-quiesce.test.d.ts +2 -0
- package/dist/test/host-quiesce.test.d.ts.map +1 -0
- package/dist/test/host-quiesce.test.js +863 -0
- package/dist/test/host-quiesce.test.js.map +1 -0
- package/dist/test/liveness-watchdog.test.js +18 -2
- package/dist/test/liveness-watchdog.test.js.map +1 -1
- package/dist/test/operator-surgery.test.d.ts +2 -0
- package/dist/test/operator-surgery.test.d.ts.map +1 -0
- package/dist/test/operator-surgery.test.js +398 -0
- package/dist/test/operator-surgery.test.js.map +1 -0
- package/dist/test/provider-acceleration-cooldown.test.js +145 -0
- package/dist/test/provider-acceleration-cooldown.test.js.map +1 -1
- package/dist/test/puppet-tool-call.test.js +5 -0
- package/dist/test/puppet-tool-call.test.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
- package/src/agent.ts +69 -0
- package/src/api/server.ts +159 -2
- package/src/api/types.ts +15 -0
- package/src/framework.ts +2155 -96
- package/src/gate/event-gate.ts +47 -4
- package/src/index.ts +5 -1
- package/src/mcpl/channel-registry.ts +495 -2
- package/src/mcpl/server-connection.ts +73 -13
- package/src/modules/history/index.ts +689 -25
- package/src/operator-log.ts +132 -0
- package/src/recovery/discord-awareness-outbox.ts +19 -0
- package/src/types/framework.ts +63 -0
- package/src/types/trace.ts +42 -0
package/src/framework.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { join } from 'node:path';
|
|
1
|
+
import { dirname, join } from 'node:path';
|
|
2
2
|
import { INLINE_WITHHELD_TEXT, classifyBlock, isInlineContradiction, referenceRegistry, referenceStubOrNull } from './mcpl/references.js';
|
|
3
3
|
import { ReferenceFetcher, DEFAULT_FETCH_MAX_BYTES, EAGER_FETCH_TIMEOUT_MS } from './mcpl/reference-fetcher.js';
|
|
4
|
-
import { appendFileSync, mkdirSync } from 'node:fs';
|
|
4
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
|
5
5
|
import { JsStore } from '@animalabs/chronicle';
|
|
6
6
|
import type { Membrane, ContentBlock, NormalizedRequest, YieldingStream, ToolResult as MembraneToolResult, ToolResultContentBlock } from '@animalabs/membrane';
|
|
7
7
|
import { MembraneError } from '@animalabs/membrane';
|
|
@@ -21,6 +21,7 @@ import type {
|
|
|
21
21
|
FrameworkConfig,
|
|
22
22
|
InferencePolicy,
|
|
23
23
|
ErrorPolicy,
|
|
24
|
+
ProviderHoldHook,
|
|
24
25
|
ErrorAction,
|
|
25
26
|
FrameworkState,
|
|
26
27
|
TraceEvent,
|
|
@@ -394,8 +395,50 @@ import {
|
|
|
394
395
|
defaultDiscordAwarenessOutboxPath,
|
|
395
396
|
extractDiscordAwarenessRefs,
|
|
396
397
|
} from './recovery/discord-awareness-outbox.js';
|
|
398
|
+
import {
|
|
399
|
+
OperatorActionError,
|
|
400
|
+
OperatorLog,
|
|
401
|
+
capIds,
|
|
402
|
+
defaultOperatorLogPath,
|
|
403
|
+
type OperatorLogEntry,
|
|
404
|
+
type OperatorLogInput,
|
|
405
|
+
type OperatorRequester,
|
|
406
|
+
} from './operator-log.js';
|
|
397
407
|
|
|
398
408
|
const FRAMEWORK_STATE_ID = 'framework/state';
|
|
409
|
+
/** Snapshot slot mirroring context writes deferred while quiesced (#122). */
|
|
410
|
+
const DEFERRED_WRITES_ID = 'framework/deferred-writes';
|
|
411
|
+
/** Above this serialized size the deferred-write mirror is skipped (kept in
|
|
412
|
+
* memory only) rather than rewriting a multi-MB snapshot per deferral. */
|
|
413
|
+
const DEFERRED_WRITES_PERSIST_CAP_BYTES = 4 * 1024 * 1024;
|
|
414
|
+
|
|
415
|
+
/** Requests that CONTINUE a turn whose token is already held — a context-
|
|
416
|
+
* budget restart, or the non-streaming path's tool-results round. The
|
|
417
|
+
* quiesce wake gate lets these through: the drain is exactly the phase
|
|
418
|
+
* where such a turn must be allowed to finish, and parking them would hold
|
|
419
|
+
* the token until the drain timed out (and `abandon` could not clear a turn
|
|
420
|
+
* with no stream). */
|
|
421
|
+
/** One entry of the deferred-write queue (see AgentFramework.deferredMessages). */
|
|
422
|
+
interface DeferredWrite {
|
|
423
|
+
id: string;
|
|
424
|
+
seq: number;
|
|
425
|
+
participant: string;
|
|
426
|
+
content: ContentBlock[];
|
|
427
|
+
metadata?: MessageMetadata;
|
|
428
|
+
forAgent?: string;
|
|
429
|
+
}
|
|
430
|
+
const bySeq = (a: { seq: number }, b: { seq: number }): number => a.seq - b.seq;
|
|
431
|
+
|
|
432
|
+
/** Stamp a deferred write's durable id into the metadata it is stored with
|
|
433
|
+
* (boot recovery dedups replays by it). Idempotent for an already-stamped
|
|
434
|
+
* message. */
|
|
435
|
+
function withDeferredWriteId(metadata: MessageMetadata | undefined, id: string): MessageMetadata {
|
|
436
|
+
return { ...(metadata ?? {}), deferredWriteId: id } as MessageMetadata;
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
function isTurnContinuation(reason: string): boolean {
|
|
440
|
+
return reason === 'context_budget_restart' || reason === 'tool_results_ready';
|
|
441
|
+
}
|
|
399
442
|
const CONVERSATION_ROUTER_STATE_ID = 'framework/conversation-router';
|
|
400
443
|
const INFERENCE_LOG_ID = 'framework/inference-log';
|
|
401
444
|
const PROCESS_LOG_ID = 'framework/process-log';
|
|
@@ -444,6 +487,106 @@ class DiscordAwarenessAccountingError extends Error {
|
|
|
444
487
|
}
|
|
445
488
|
}
|
|
446
489
|
|
|
490
|
+
/**
|
|
491
|
+
* Feasibility verdict for a runtime-settings patch, derived through the SAME
|
|
492
|
+
* settings→config mapping the live apply path uses (`Agent.planRuntimeSettings`
|
|
493
|
+
* → `previewContextSettings`). The `effective` verdict is for the budget the
|
|
494
|
+
* next compile actually plans at — for a paced descent that is the UNCHANGED
|
|
495
|
+
* live budget, with the descent target reported separately as `advisory`.
|
|
496
|
+
*/
|
|
497
|
+
export interface RuntimeSettingsPreview {
|
|
498
|
+
/** False when no verdict could be computed (no strategy support, preview
|
|
499
|
+
* overlap, stale branch generation, …) — see `reason`. Never silently
|
|
500
|
+
* swallowed: an unavailable preview is an answer, not an error. */
|
|
501
|
+
available: boolean;
|
|
502
|
+
reason?:
|
|
503
|
+
| 'no_preview_support'
|
|
504
|
+
| 'preview_in_flight'
|
|
505
|
+
| 'branch_generation_changed'
|
|
506
|
+
| 'no_adaptive_resolution'
|
|
507
|
+
| string;
|
|
508
|
+
path: 'immediate' | 'paced' | 'none';
|
|
509
|
+
/** The budget compiles currently plan at (baseline of the patch). */
|
|
510
|
+
liveBudgetTokens: number;
|
|
511
|
+
/** Verdict at the effective compile budget (spread of the strategy's
|
|
512
|
+
* PreviewResult: finalTokens, fits, headTokens, tailTokens, middleTokens,
|
|
513
|
+
* deepestLevel, exhausted, …). */
|
|
514
|
+
effective?: { budgetTokens: number; fits: boolean; finalTokens: number } & Record<string, unknown>;
|
|
515
|
+
/** Best-effort verdict at a paced-descent target. Advisory only — a target
|
|
516
|
+
* below the folded floor does not block the patch; the descent simply
|
|
517
|
+
* converges as far as the floor allows. */
|
|
518
|
+
advisory?: { targetTokens: number; fits: boolean } & Record<string, unknown>;
|
|
519
|
+
transition: 'stable' | 'converging' | 'blocked';
|
|
520
|
+
transitionReason?: string;
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
/**
|
|
524
|
+
* Thrown by `updateAgentRuntimeSettings` when an immediate budget change would
|
|
525
|
+
* put the agent in an un-compilable layout: the folded floor renders more
|
|
526
|
+
* tokens than the hard budget admits, so every subsequent compile would
|
|
527
|
+
* OverBudget-wedge the agent (2026-08-21: mythos, 550k→260k set live against a
|
|
528
|
+
* ~450k floor — hours of hard-down). Overridable with `{ allowInfeasible: true }`.
|
|
529
|
+
* Paced descents never throw this: a non-immediate decrease leaves the compile
|
|
530
|
+
* budget untouched and converges only as far as the floor allows.
|
|
531
|
+
*/
|
|
532
|
+
export class BudgetPreflightError extends Error {
|
|
533
|
+
constructor(
|
|
534
|
+
message: string,
|
|
535
|
+
readonly preview: RuntimeSettingsPreview,
|
|
536
|
+
) {
|
|
537
|
+
super(message);
|
|
538
|
+
this.name = 'BudgetPreflightError';
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
/** Operator-facing snapshot of the host's quiesce state (issue #122). */
|
|
543
|
+
export interface HostModeStatus {
|
|
544
|
+
quiesced: boolean;
|
|
545
|
+
reason?: string;
|
|
546
|
+
since?: number;
|
|
547
|
+
/** True when no turn is alive (activeTurnTokens empty — the token spans
|
|
548
|
+
* dequeue → settled teardown, strictly wider than activeStreams). */
|
|
549
|
+
drained: boolean;
|
|
550
|
+
activeTurns: number;
|
|
551
|
+
/** Wakes that own provider admission while waiting for an in-flight
|
|
552
|
+
* auxiliary call — no turn token yet, but a turn is one settle away. The
|
|
553
|
+
* admission continuation rechecks quiesce and requeues them; `drained`
|
|
554
|
+
* is false until it has. */
|
|
555
|
+
parkedAdmissions: number;
|
|
556
|
+
/** Inference requests parked by the quiesce wake gate (while quiesced), or
|
|
557
|
+
* simply queued (while serving — non-zero right after a resume until the
|
|
558
|
+
* scheduler's next pass consumes them). Continuations of a held turn
|
|
559
|
+
* (budget restart, tool results) are never counted. */
|
|
560
|
+
gatedRequests: number;
|
|
561
|
+
/** Background code-execution scripts still running. Quiesce does NOT stop
|
|
562
|
+
* them — they hold no turn token, so drain doesn't wait for them. Reported
|
|
563
|
+
* so the operator sees what is still acting during the window. */
|
|
564
|
+
backgroundScripts: number;
|
|
565
|
+
/** Context writes (module events, api message.send) withheld because the
|
|
566
|
+
* host is quiesced. Persisted while quiesced and flushed by resume(). */
|
|
567
|
+
deferredWrites: number;
|
|
568
|
+
/** Agents whose turn `abandon` could NOT cancel: the turn token is held but
|
|
569
|
+
* no stream exists yet (still in hooks/compile) or ever will (puppet tool
|
|
570
|
+
* turns). Present only after an abandon that left such turns behind. */
|
|
571
|
+
unabandonable?: string[];
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
/**
|
|
575
|
+
* Thrown by `resume()` when the current runtime settings do not compile for
|
|
576
|
+
* one or more agents — returning to service would OverBudget-wedge them on
|
|
577
|
+
* the first wake. Drain quarantine / advance merges to lower the floor, or
|
|
578
|
+
* pass `{ force: true }` after deciding the verdicts are acceptable.
|
|
579
|
+
*/
|
|
580
|
+
export class ResumeBlockedError extends Error {
|
|
581
|
+
constructor(
|
|
582
|
+
message: string,
|
|
583
|
+
readonly verdicts: Array<{ agentName: string; preview: RuntimeSettingsPreview }>,
|
|
584
|
+
) {
|
|
585
|
+
super(message);
|
|
586
|
+
this.name = 'ResumeBlockedError';
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
|
|
447
590
|
interface RedoEntry {
|
|
448
591
|
branchName: string;
|
|
449
592
|
checkpoint: TurnCheckpoint;
|
|
@@ -466,6 +609,85 @@ class DefaultInferencePolicy implements InferencePolicy {
|
|
|
466
609
|
}
|
|
467
610
|
}
|
|
468
611
|
|
|
612
|
+
/** Minimal read surface the live-surgery helpers need from a ContextManager. */
|
|
613
|
+
interface SurgeryReadableCm {
|
|
614
|
+
getMessageCount(): number;
|
|
615
|
+
getMessageWindow(
|
|
616
|
+
offset: number,
|
|
617
|
+
limit: number,
|
|
618
|
+
opts?: { resolveBlobs?: boolean; alignToBodyGroups?: boolean },
|
|
619
|
+
): { messages: Array<{ id: unknown; bodyGroupId?: string }>; startIndex: number };
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
const SURGERY_SCAN_WINDOW = 500;
|
|
623
|
+
|
|
624
|
+
/**
|
|
625
|
+
* Slot index of `messageId` on the active branch, or -1. Scans from the tail
|
|
626
|
+
* in blob-free windows: operator targets are almost always recent, and a
|
|
627
|
+
* full `getAllMessages()` re-inflates every attachment on the branch.
|
|
628
|
+
*/
|
|
629
|
+
function locateMessageIndex(cm: SurgeryReadableCm, messageId: string): number {
|
|
630
|
+
const hit = locateMessageIndices(cm, new Set([messageId])).get(messageId);
|
|
631
|
+
return hit ? hit.index : -1;
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
/** Tail-first scan resolving many ids at once; stops as soon as all are found. */
|
|
635
|
+
function locateMessageIndices(
|
|
636
|
+
cm: SurgeryReadableCm,
|
|
637
|
+
ids: Set<string>,
|
|
638
|
+
): Map<string, { index: number; bodyGroupId?: string }> {
|
|
639
|
+
const found = new Map<string, { index: number; bodyGroupId?: string }>();
|
|
640
|
+
const total = cm.getMessageCount();
|
|
641
|
+
for (let end = total; end > 0 && found.size < ids.size; end -= SURGERY_SCAN_WINDOW) {
|
|
642
|
+
const start = Math.max(0, end - SURGERY_SCAN_WINDOW);
|
|
643
|
+
const { messages, startIndex } = cm.getMessageWindow(start, end - start, { resolveBlobs: false });
|
|
644
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
645
|
+
const id = String(messages[i].id);
|
|
646
|
+
if (ids.has(id) && !found.has(id)) {
|
|
647
|
+
found.set(id, {
|
|
648
|
+
index: startIndex + i,
|
|
649
|
+
...(messages[i].bodyGroupId ? { bodyGroupId: messages[i].bodyGroupId } : {}),
|
|
650
|
+
});
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
return found;
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
/**
|
|
658
|
+
* Inclusive slot range of the body-group run containing `index` (the slot
|
|
659
|
+
* itself when it is not sharded). Delegates to the store's own
|
|
660
|
+
* `alignToBodyGroups` edge walk — one contiguity assumption, not two.
|
|
661
|
+
*/
|
|
662
|
+
function bodyGroupRun(
|
|
663
|
+
cm: SurgeryReadableCm,
|
|
664
|
+
index: number,
|
|
665
|
+
): { from: number; to: number; fromId: string; toId: string; group: boolean } {
|
|
666
|
+
const win = cm.getMessageWindow(index, 1, { alignToBodyGroups: true, resolveBlobs: false });
|
|
667
|
+
const from = win.startIndex;
|
|
668
|
+
const to = win.startIndex + win.messages.length - 1;
|
|
669
|
+
return {
|
|
670
|
+
from,
|
|
671
|
+
to,
|
|
672
|
+
fromId: String(win.messages[0].id),
|
|
673
|
+
toId: String(win.messages[win.messages.length - 1].id),
|
|
674
|
+
group: win.messages[0].bodyGroupId !== undefined,
|
|
675
|
+
};
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
function describeRequester(r: OperatorRequester | undefined): string {
|
|
679
|
+
if (!r) return 'unknown';
|
|
680
|
+
return `${r.name ?? r.id ?? 'unknown'} (${r.via})`;
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
function hostCommandRequester(serverId: string, params: HostCommandParams): OperatorRequester {
|
|
684
|
+
return {
|
|
685
|
+
via: `host-command:${serverId}`,
|
|
686
|
+
...(params.requesterName ? { name: params.requesterName } : {}),
|
|
687
|
+
...(params.requesterId ? { id: params.requesterId } : {}),
|
|
688
|
+
};
|
|
689
|
+
}
|
|
690
|
+
|
|
469
691
|
function isPermanentDiscordReactionFailure(message: string): boolean {
|
|
470
692
|
return /unknown message|unknown channel|missing access|missing permissions|missing permission|cannot access|channel .* not found|message .* not found/i
|
|
471
693
|
.test(message);
|
|
@@ -523,6 +745,12 @@ interface ProviderAccelerationCooldown {
|
|
|
523
745
|
heldRequests: InferenceRequest[];
|
|
524
746
|
reason: string;
|
|
525
747
|
failures: number;
|
|
748
|
+
/** Set on a host-requested hold (config.providerHold): the error to
|
|
749
|
+
* re-present to the hook when this slice expires. */
|
|
750
|
+
hostHoldError?: Error;
|
|
751
|
+
/** Armed by an auxiliary (compression) failure with no primary involved:
|
|
752
|
+
* release reopens auxiliary admission without synthesising an inference. */
|
|
753
|
+
auxiliaryOrigin?: boolean;
|
|
526
754
|
}
|
|
527
755
|
interface ProviderAccelerationRecovery {
|
|
528
756
|
startedAt: number;
|
|
@@ -614,6 +842,14 @@ interface HostCommandParams {
|
|
|
614
842
|
maxRewinds?: number;
|
|
615
843
|
/** For the `unstick` command: raw channel id to post the outcome report to. */
|
|
616
844
|
channelId?: string;
|
|
845
|
+
/** For the `quiesce` command: operator-facing reason recorded in the mode. */
|
|
846
|
+
reason?: string;
|
|
847
|
+
/** For the `quiesce` command: drain window in ms (clamped to [1s, 10m]). */
|
|
848
|
+
timeoutMs?: number;
|
|
849
|
+
/** For the `quiesce` command: cancel undrained turns after the window. */
|
|
850
|
+
abandon?: boolean;
|
|
851
|
+
/** For the `resume` command: override a failing feasibility verdict. */
|
|
852
|
+
force?: boolean;
|
|
617
853
|
requesterId?: string;
|
|
618
854
|
requesterName?: string;
|
|
619
855
|
}
|
|
@@ -693,6 +929,17 @@ export class AgentFramework {
|
|
|
693
929
|
private activeTriggerChannels: Map<string, string> = new Map();
|
|
694
930
|
private running = false;
|
|
695
931
|
private loopPromise: Promise<void> | null = null;
|
|
932
|
+
/** Quiesce/maintenance mode (issue #122). DELIBERATELY separate from
|
|
933
|
+
* `running`, which is overloaded as the runLoop condition, the maintenance
|
|
934
|
+
* admission guard, AND the per-tick loop condition — expressing "paused" by
|
|
935
|
+
* clearing `running` would kill the very maintenance machinery quiesce
|
|
936
|
+
* exists to keep hot, plus store sync and tool-result processing. While
|
|
937
|
+
* quiesced: runLoop, syncTimer, maintenanceTimer, and the watchdog all keep
|
|
938
|
+
* running; only new turns (wake gate in processInferenceRequests) and MCPL
|
|
939
|
+
* data planes are held. */
|
|
940
|
+
private quiesced = false;
|
|
941
|
+
private quiesceReason?: string;
|
|
942
|
+
private quiescedAt?: number;
|
|
696
943
|
private traceListeners: TraceEventListener[] = [];
|
|
697
944
|
private syncIntervalMs: number;
|
|
698
945
|
private syncTimer: ReturnType<typeof setInterval> | null = null;
|
|
@@ -708,6 +955,7 @@ export class AgentFramework {
|
|
|
708
955
|
private providerAccelerationLastRecovery: Map<string, ProviderAccelerationReceipt> = new Map();
|
|
709
956
|
private providerAccelerationDefaultCooldownMs = PROVIDER_ACCELERATION_DEFAULT_COOLDOWN_MS;
|
|
710
957
|
private providerAccelerationJitterMs = PROVIDER_ACCELERATION_JITTER_MS;
|
|
958
|
+
private providerHoldHook: ProviderHoldHook | undefined;
|
|
711
959
|
private providerAdmissionClosed = false;
|
|
712
960
|
/** Last time we reported stale (busy-requeued) inference requests, per agent. */
|
|
713
961
|
private staleWarnAt = new Map<string, number>();
|
|
@@ -797,7 +1045,7 @@ export class AgentFramework {
|
|
|
797
1045
|
* `inference:exhausted` (which also pollutes the failure streak). Kept
|
|
798
1046
|
* separate from ephemeralRuns deliberately: endTurn/budget cancels happen
|
|
799
1047
|
* for resident agents too, and the key is per-stream, not per-agent. */
|
|
800
|
-
private frameworkCancelledStreams: Map<string, 'turn_ended' | 'budget_restart'> = new Map();
|
|
1048
|
+
private frameworkCancelledStreams: Map<string, 'turn_ended' | 'budget_restart' | 'quiesce_abandoned'> = new Map();
|
|
801
1049
|
/** Active runEphemeralToCompletion runs, keyed by agent name. */
|
|
802
1050
|
private ephemeralRuns: Map<string, EphemeralRun> = new Map();
|
|
803
1051
|
/** Ephemeral namespaces/names are single-generation for this framework
|
|
@@ -872,7 +1120,42 @@ export class AgentFramework {
|
|
|
872
1120
|
/** Tune-out coordinator (issue #77); non-null iff subconscious + channels. */
|
|
873
1121
|
private tuneOutCoordinator: TuneOutCoordinator | null = null;
|
|
874
1122
|
|
|
1123
|
+
/** Agents an abandon could not cancel (token held, no stream) — surfaced
|
|
1124
|
+
* in HostModeStatus until the next quiesce/resume. */
|
|
1125
|
+
private lastUnabandonable: string[] = [];
|
|
1126
|
+
/** True while the deferred-write queue has a persisted mirror (quiesced). */
|
|
1127
|
+
private deferredWritesPersisted = false;
|
|
1128
|
+
private deferredWritesCapWarned = false;
|
|
1129
|
+
/** Branch-independent recovery files (see FrameworkConfig.hostModePath /
|
|
1130
|
+
* deferredWritesPath). Undefined → branch-local slot fallback. */
|
|
1131
|
+
private hostModePath: string | undefined;
|
|
1132
|
+
private deferredWritesPath: string | undefined;
|
|
1133
|
+
private recoveryFallbackWarned = false;
|
|
1134
|
+
/**
|
|
1135
|
+
* Deferred writes that have been handed to a context manager but whose
|
|
1136
|
+
* chronicle state is NOT yet durably synced. They stay in the durable
|
|
1137
|
+
* recovery queue until `ackDeferredWrites()` syncs the store — chronicle
|
|
1138
|
+
* persists the slot-chain head only on sync(), so acknowledging on append
|
|
1139
|
+
* alone would let a hard exit forget an accepted message.
|
|
1140
|
+
*/
|
|
1141
|
+
private unackedDeferredWrites: DeferredWrite[] = [];
|
|
1142
|
+
/** Monotonic order stamp for deferred writes: the durable queue is always
|
|
1143
|
+
* written, restored, drained and flushed in `seq` order, whatever the
|
|
1144
|
+
* pending/un-acked split — a re-deferred entry keeps its place. */
|
|
1145
|
+
private deferredSeq = 0;
|
|
1146
|
+
/**
|
|
1147
|
+
* Per target agent: the store's message count when the OLDEST currently
|
|
1148
|
+
* un-acked entry for it was handed off. Persisted with the queue so a boot
|
|
1149
|
+
* after an interrupted flush knows exactly which slot range the batch
|
|
1150
|
+
* could occupy and dedups against all of it — never a fixed tail.
|
|
1151
|
+
*/
|
|
1152
|
+
private deferredScanFrom = new Map<string, number>();
|
|
875
1153
|
private deferredMessages: Array<{
|
|
1154
|
+
/** Durable identity for per-message flush acknowledgement: a flush
|
|
1155
|
+
* interrupted by a crash replays only the messages not yet acked. */
|
|
1156
|
+
id: string;
|
|
1157
|
+
/** Order stamp (see deferredSeq). */
|
|
1158
|
+
seq: number;
|
|
876
1159
|
participant: string;
|
|
877
1160
|
content: ContentBlock[];
|
|
878
1161
|
metadata?: MessageMetadata;
|
|
@@ -998,6 +1281,16 @@ export class AgentFramework {
|
|
|
998
1281
|
private discordAwarenessOutbox: DiscordAwarenessOutbox | null = null;
|
|
999
1282
|
private discordAwarenessEmoji = DEFAULT_DISCORD_AWARENESS_EMOJI;
|
|
1000
1283
|
private discordAwarenessDeadlineMs = DEFAULT_DISCORD_AWARENESS_DEADLINE_MS;
|
|
1284
|
+
/** Durable JSONL record of operator-initiated mutations (see operator-log.ts). */
|
|
1285
|
+
private readonly operatorLog: OperatorLog;
|
|
1286
|
+
/**
|
|
1287
|
+
* Store-wide admission barrier while a live surgery (rollback/suppress)
|
|
1288
|
+
* holds the store. Per-agent turn tokens cover the agents registered when
|
|
1289
|
+
* the reservation was taken; this flag covers everyone else — an ephemeral
|
|
1290
|
+
* agent admitted mid-switch, a wake for an agent created later — at the
|
|
1291
|
+
* scheduler, ephemeral admission and puppet entry points.
|
|
1292
|
+
*/
|
|
1293
|
+
private surgeryHold: { verb: string; agentName: string; since: number } | null = null;
|
|
1001
1294
|
/** Serialize per-server drains so reconnect and an online undo cannot race. */
|
|
1002
1295
|
private discordAwarenessDrains: Map<string, Promise<DiscordAwarenessDrainOutcome>> = new Map();
|
|
1003
1296
|
/** Framework-global inference gate; older generations cannot release it. */
|
|
@@ -1026,7 +1319,9 @@ export class AgentFramework {
|
|
|
1026
1319
|
discordAwarenessOutbox: DiscordAwarenessOutbox | null,
|
|
1027
1320
|
discordAwarenessEmoji: string,
|
|
1028
1321
|
discordAwarenessDeadlineMs: number,
|
|
1322
|
+
operatorLog: OperatorLog = new OperatorLog(undefined),
|
|
1029
1323
|
) {
|
|
1324
|
+
this.operatorLog = operatorLog;
|
|
1030
1325
|
this.store = store;
|
|
1031
1326
|
this.ownsStore = ownsStore;
|
|
1032
1327
|
this.membrane = membrane;
|
|
@@ -1088,6 +1383,11 @@ export class AgentFramework {
|
|
|
1088
1383
|
} catch {
|
|
1089
1384
|
// Already registered
|
|
1090
1385
|
}
|
|
1386
|
+
try {
|
|
1387
|
+
store.registerState({ id: DEFERRED_WRITES_ID, strategy: 'snapshot' });
|
|
1388
|
+
} catch {
|
|
1389
|
+
// Already registered
|
|
1390
|
+
}
|
|
1091
1391
|
|
|
1092
1392
|
try {
|
|
1093
1393
|
store.registerState({
|
|
@@ -1132,6 +1432,18 @@ export class AgentFramework {
|
|
|
1132
1432
|
? new DiscordAwarenessOutbox(discordAwarenessOutboxPath)
|
|
1133
1433
|
: null;
|
|
1134
1434
|
|
|
1435
|
+
const operatorLogPath = config.operatorLogPath === false
|
|
1436
|
+
? undefined
|
|
1437
|
+
: config.operatorLogPath
|
|
1438
|
+
?? (config.storePath ? defaultOperatorLogPath(config.storePath) : undefined);
|
|
1439
|
+
if (operatorLogPath === undefined && config.operatorLogPath !== false) {
|
|
1440
|
+
// An audit log that is silently off is worse than none: say so once.
|
|
1441
|
+
console.error(
|
|
1442
|
+
'[operator-log] disabled: no storePath and no operatorLogPath configured — ' +
|
|
1443
|
+
'operator actions will only be visible as operator:action traces',
|
|
1444
|
+
);
|
|
1445
|
+
}
|
|
1446
|
+
|
|
1135
1447
|
const framework = new AgentFramework(
|
|
1136
1448
|
store,
|
|
1137
1449
|
ownsStore,
|
|
@@ -1146,7 +1458,9 @@ export class AgentFramework {
|
|
|
1146
1458
|
discordAwarenessOutbox,
|
|
1147
1459
|
config.discordAwarenessEmoji ?? DEFAULT_DISCORD_AWARENESS_EMOJI,
|
|
1148
1460
|
normalizeDiscordAwarenessDeadline(config.discordAwarenessDeadlineMs),
|
|
1461
|
+
new OperatorLog(operatorLogPath),
|
|
1149
1462
|
);
|
|
1463
|
+
framework.providerHoldHook = config.providerHold;
|
|
1150
1464
|
|
|
1151
1465
|
// If an offline recovery process crashed after switching Chronicle but
|
|
1152
1466
|
// before committing its prepared marker batch, the active branch is the
|
|
@@ -1287,6 +1601,61 @@ export class AgentFramework {
|
|
|
1287
1601
|
framework.toolResultInlineMaxCharsOverride.set(agentName, cap);
|
|
1288
1602
|
}
|
|
1289
1603
|
|
|
1604
|
+
// Restore persisted quiesce mode (issue #122) BEFORE initializeMcpl: the
|
|
1605
|
+
// flag must be set before any data-plane barrier completion can run, or
|
|
1606
|
+
// the startup funnel would open the data planes on a host that shut down
|
|
1607
|
+
// mid-maintenance. Staged connections boot with both planes closed, so a
|
|
1608
|
+
// quiesced boot needs no re-pause — completeMcplDataPlaneGate consults
|
|
1609
|
+
// the flag and holds data planes (control planes come up normally). The
|
|
1610
|
+
// gate already exists at this point, so the suppression is wired here too.
|
|
1611
|
+
//
|
|
1612
|
+
// Both records live OUTSIDE branch history (recovery/ files next to the
|
|
1613
|
+
// store): a historical rollback must not be able to erase the marker of
|
|
1614
|
+
// the very surgery it belongs to, nor orphan the writes deferred by it.
|
|
1615
|
+
framework.hostModePath = config.hostModePath
|
|
1616
|
+
?? (config.storePath ? join(config.storePath, 'recovery', 'host-mode.json') : undefined);
|
|
1617
|
+
framework.deferredWritesPath = config.deferredWritesPath
|
|
1618
|
+
?? (config.storePath ? join(config.storePath, 'recovery', 'deferred-writes.json') : undefined);
|
|
1619
|
+
{
|
|
1620
|
+
const hostMode = framework.readHostMode();
|
|
1621
|
+
// Deferred writes are recovered REGARDLESS of the mode flag: a crash
|
|
1622
|
+
// between a resume's flag clear and the end of its flush must not
|
|
1623
|
+
// strand accepted messages. Per-message acks mean only the remainder
|
|
1624
|
+
// replays. Serving boot with a remainder → flush it right now (no turn
|
|
1625
|
+
// is alive yet, so this is a safe boundary).
|
|
1626
|
+
const restoredWrites = framework.restorePersistedDeferredWrites();
|
|
1627
|
+
if (!hostMode?.quiesced && restoredWrites > 0) {
|
|
1628
|
+
console.error(
|
|
1629
|
+
`[host-mode] ${restoredWrites} deferred context write(s) found at a serving boot ` +
|
|
1630
|
+
`(an earlier resume did not finish its flush) — landing them now`,
|
|
1631
|
+
);
|
|
1632
|
+
await framework.flushDeferredWrites('boot-recovery');
|
|
1633
|
+
}
|
|
1634
|
+
if (hostMode?.quiesced) {
|
|
1635
|
+
framework.quiesced = true;
|
|
1636
|
+
framework.quiesceReason = hostMode.reason;
|
|
1637
|
+
framework.quiescedAt = hostMode.since;
|
|
1638
|
+
framework.eventGate?.setQuiesced(true);
|
|
1639
|
+
console.error(
|
|
1640
|
+
`[host-mode] ============================================================\n` +
|
|
1641
|
+
`[host-mode] BOOTING QUIESCED (persisted${hostMode.reason ? `: ${hostMode.reason}` : ''}, ` +
|
|
1642
|
+
`since ${hostMode.since ? new Date(hostMode.since).toISOString() : 'unknown'}).\n` +
|
|
1643
|
+
`[host-mode] No wakes will start turns and MCPL data planes stay paused\n` +
|
|
1644
|
+
`[host-mode] until resume() — via host/command, the API server, or the\n` +
|
|
1645
|
+
`[host-mode] framework API. Wakes parked before the restart were in-memory\n` +
|
|
1646
|
+
`[host-mode] and are gone; recurring sources re-deliver on their own cadence.\n` +
|
|
1647
|
+
`[host-mode] ${restoredWrites} deferred context write(s) restored from the store;\n` +
|
|
1648
|
+
`[host-mode] they land at resume().\n` +
|
|
1649
|
+
`[host-mode] ============================================================`,
|
|
1650
|
+
);
|
|
1651
|
+
framework.emitTrace({
|
|
1652
|
+
type: 'host:quiesced_boot',
|
|
1653
|
+
...(hostMode.reason ? { reason: hostMode.reason } : {}),
|
|
1654
|
+
...(hostMode.since !== undefined ? { since: hostMode.since } : {}),
|
|
1655
|
+
});
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1290
1659
|
// Initialize MCPL subsystems if configured
|
|
1291
1660
|
if (config.mcplServers && config.mcplServers.length > 0) {
|
|
1292
1661
|
// Validate tool prefixes: no collisions with module names or between servers
|
|
@@ -1412,6 +1781,9 @@ export class AgentFramework {
|
|
|
1412
1781
|
*/
|
|
1413
1782
|
async stop(): Promise<void> {
|
|
1414
1783
|
this.running = false;
|
|
1784
|
+
// Flushed-but-unsynced deferred writes: sync and ack now, while the
|
|
1785
|
+
// store is still open, rather than leaving them to a reboot replay.
|
|
1786
|
+
this.ackDeferredWrites();
|
|
1415
1787
|
this.providerAdmissionClosed = true;
|
|
1416
1788
|
this.queue.close();
|
|
1417
1789
|
this.tuneOutCoordinator?.stop();
|
|
@@ -1552,6 +1924,14 @@ export class AgentFramework {
|
|
|
1552
1924
|
}
|
|
1553
1925
|
gate.auxiliaryInFlight++;
|
|
1554
1926
|
try { return await run(); }
|
|
1927
|
+
catch (error) {
|
|
1928
|
+
// Compression can be the first call to find a spent quota (idle agent,
|
|
1929
|
+
// scheduled maintenance). Arm the host hold so later passes wait at the
|
|
1930
|
+
// gate instead of retrying into it; this pass still fails to its caller.
|
|
1931
|
+
const agent = this.agents.get(agentName);
|
|
1932
|
+
if (agent && error instanceof Error) this.holdProviderAcceleration(agent, error, undefined, true);
|
|
1933
|
+
throw error;
|
|
1934
|
+
}
|
|
1555
1935
|
finally {
|
|
1556
1936
|
gate.auxiliaryInFlight = Math.max(0, gate.auxiliaryInFlight - 1);
|
|
1557
1937
|
if (gate.auxiliaryInFlight === 0) for (const resolve of gate.idleWaiters.splice(0)) resolve();
|
|
@@ -1578,27 +1958,75 @@ export class AgentFramework {
|
|
|
1578
1958
|
return a.agentName === b.agentName && a.reason === b.reason && a.source === b.source &&
|
|
1579
1959
|
a.timestamp === b.timestamp && a.channelId === b.channelId;
|
|
1580
1960
|
}
|
|
1581
|
-
private
|
|
1582
|
-
if (this.
|
|
1583
|
-
|
|
1961
|
+
private consultProviderHold(error: Error, agentName: string): { holdMs: number; reason: string } | undefined {
|
|
1962
|
+
if (!this.providerHoldHook) return undefined;
|
|
1963
|
+
try {
|
|
1964
|
+
const hold = this.providerHoldHook(error, agentName, { model: this.agents.get(agentName)?.model });
|
|
1965
|
+
if (!hold || !Number.isFinite(hold.holdMs) || hold.holdMs <= 0) return undefined;
|
|
1966
|
+
return { holdMs: Math.min(PROVIDER_ACCELERATION_MAX_COOLDOWN_MS, Math.max(1_000, hold.holdMs)),
|
|
1967
|
+
reason: typeof hold.reason === 'string' && hold.reason ? hold.reason : error.message };
|
|
1968
|
+
} catch (err) {
|
|
1969
|
+
console.error(`[provider-cooldown] providerHold hook threw for ${agentName}; treating as no hold:`, err);
|
|
1970
|
+
return undefined;
|
|
1971
|
+
}
|
|
1972
|
+
}
|
|
1973
|
+
/** A host hold whose slice expired: ask the host again before spending an
|
|
1974
|
+
* inference on it. True = extended in place (held requests stay held). */
|
|
1975
|
+
private extendHostProviderHold(agentName: string, cooldown: ProviderAccelerationCooldown): boolean {
|
|
1976
|
+
if (!cooldown.hostHoldError) return false;
|
|
1977
|
+
const hold = this.consultProviderHold(cooldown.hostHoldError, agentName);
|
|
1978
|
+
if (!hold) return false;
|
|
1979
|
+
clearTimeout(cooldown.timer);
|
|
1980
|
+
cooldown.until = Date.now() + hold.holdMs;
|
|
1981
|
+
cooldown.timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agentName), hold.holdMs);
|
|
1982
|
+
cooldown.timer.unref?.();
|
|
1983
|
+
if (hold.reason !== cooldown.reason) {
|
|
1984
|
+
cooldown.reason = hold.reason;
|
|
1985
|
+
console.error(`[provider-cooldown] agent=${agentName} host hold extended — ${hold.reason}`);
|
|
1986
|
+
}
|
|
1987
|
+
return true;
|
|
1988
|
+
}
|
|
1989
|
+
private holdProviderAcceleration(agent: Agent, error: Error, trigger?: InferenceRequest, auxiliary = false): boolean {
|
|
1990
|
+
if (this.ephemeralRuns.has(agent.name) || this.conversationAgentHomes.has(agent.name)) return false;
|
|
1991
|
+
// The host knows more than the message wording does: ask it first. An
|
|
1992
|
+
// auxiliary failure arms host holds only — the built-in acceleration
|
|
1993
|
+
// cooldown stays a primary-path mechanism.
|
|
1994
|
+
const hostHold = this.consultProviderHold(error, agent.name);
|
|
1995
|
+
const acceleration = !hostHold && !auxiliary && isOrganizationAccelerationRateLimit(error);
|
|
1996
|
+
if (!acceleration && !hostHold) return false;
|
|
1997
|
+
const now = Date.now(); const delayMs = hostHold ? hostHold.holdMs : this.accelerationCooldownMs(agent.name, error as MembraneError);
|
|
1584
1998
|
const existing = this.providerAccelerationCooldowns.get(agent.name);
|
|
1585
1999
|
const held = existing?.heldRequests ?? [];
|
|
1586
2000
|
if (trigger && !held.some((r) => this.sameInferenceRequest(r, trigger))) held.push(trigger);
|
|
1587
2001
|
if (existing) clearTimeout(existing.timer);
|
|
1588
2002
|
const timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agent.name), delayMs); timer.unref?.();
|
|
1589
2003
|
this.providerAccelerationCooldowns.set(agent.name, { startedAt: existing?.startedAt ?? now,
|
|
1590
|
-
until: now + delayMs, timer, heldRequests: held, reason: error.message,
|
|
1591
|
-
failures: (existing?.failures ?? 0) + 1 })
|
|
2004
|
+
until: now + delayMs, timer, heldRequests: held, reason: hostHold?.reason ?? error.message,
|
|
2005
|
+
failures: (existing?.failures ?? 0) + 1, ...(hostHold ? { hostHoldError: error } : {}),
|
|
2006
|
+
...(auxiliary && (!existing || existing.auxiliaryOrigin) ? { auxiliaryOrigin: true } : {}) });
|
|
2007
|
+
if (auxiliary && !existing) {
|
|
2008
|
+
console.error(`[provider-cooldown] agent=${agent.name} host hold (${hostHold!.reason}) from an auxiliary call — ` +
|
|
2009
|
+
`holding primary/auxiliary for ${delayMs}ms`);
|
|
2010
|
+
return true;
|
|
2011
|
+
}
|
|
1592
2012
|
const gate = this.providerGate(agent.name); gate.primaryPending = true;
|
|
1593
2013
|
this.providerAccelerationRecoveries.set(agent.name, { startedAt: existing?.startedAt ?? now,
|
|
1594
2014
|
failures: (existing?.failures ?? 0) + 1, heldRequests: held.length, reason: error.message });
|
|
1595
|
-
console.error(`[provider-cooldown] agent=${agent.name}
|
|
2015
|
+
console.error(`[provider-cooldown] agent=${agent.name} ` +
|
|
2016
|
+
`${hostHold ? `host hold (${hostHold.reason})` : 'organization acceleration 429'} — ` +
|
|
1596
2017
|
`holding primary/auxiliary for ${delayMs}ms; ${held.length} request(s) retained`);
|
|
1597
2018
|
return true;
|
|
1598
2019
|
}
|
|
1599
2020
|
private releaseProviderAccelerationCooldown(agentName: string): void {
|
|
1600
2021
|
const cooldown = this.providerAccelerationCooldowns.get(agentName); if (!cooldown) return;
|
|
2022
|
+
if (this.extendHostProviderHold(agentName, cooldown)) return;
|
|
1601
2023
|
clearTimeout(cooldown.timer); this.providerAccelerationCooldowns.delete(agentName);
|
|
2024
|
+
if (cooldown.auxiliaryOrigin && cooldown.heldRequests.length === 0) {
|
|
2025
|
+
// Nobody asked for an inference while parked: just reopen the gate.
|
|
2026
|
+
console.error(`[provider-cooldown] agent=${agentName} auxiliary hold released after ${Date.now() - cooldown.startedAt}ms`);
|
|
2027
|
+
this.flushAuxiliaryAdmission(agentName);
|
|
2028
|
+
return;
|
|
2029
|
+
}
|
|
1602
2030
|
const recovery = this.providerAccelerationRecoveries.get(agentName);
|
|
1603
2031
|
if (recovery) { recovery.releasedAt = Date.now(); recovery.heldRequests = cooldown.heldRequests.length; }
|
|
1604
2032
|
const requests = cooldown.heldRequests.length > 0 ? cooldown.heldRequests : [{ agentName,
|
|
@@ -2166,21 +2594,74 @@ export class AgentFramework {
|
|
|
2166
2594
|
updateAgentRuntimeSettings(
|
|
2167
2595
|
agentName: string,
|
|
2168
2596
|
patch: AgentRuntimeSettingsPatch,
|
|
2169
|
-
opts?: { persist?: boolean },
|
|
2597
|
+
opts?: { persist?: boolean; allowInfeasible?: boolean; requester?: OperatorRequester; note?: string },
|
|
2170
2598
|
): AgentRuntimeSettingsSnapshot {
|
|
2171
2599
|
const agent = this.agents.get(agentName);
|
|
2172
2600
|
if (!agent) throw new Error(`Unknown agent: ${agentName}`);
|
|
2601
|
+
// Feasibility preflight — only a budget-bearing patch can wedge, and only
|
|
2602
|
+
// on the immediate path (a paced descent leaves the compile budget alone
|
|
2603
|
+
// and converges only as far as the folded floor allows). Boot restore
|
|
2604
|
+
// deliberately bypasses this wrapper (createAgent → restoreRuntimeSettings),
|
|
2605
|
+
// so a persisted-but-now-infeasible budget can never brick startup.
|
|
2606
|
+
if (patch.contextBudgetTokens !== undefined) {
|
|
2607
|
+
const preview = this.previewAgentRuntimeSettings(agentName, patch);
|
|
2608
|
+
if (preview.available && preview.effective) {
|
|
2609
|
+
const e = preview.effective;
|
|
2610
|
+
// Block only a LOWERING that doesn't fit. A no-op rewrite or an
|
|
2611
|
+
// increase on an already-over-floor agent never makes things worse —
|
|
2612
|
+
// refusing those would throw on exactly the wedged population the
|
|
2613
|
+
// guard exists to protect (and block stepwise remediation).
|
|
2614
|
+
const lowering = patch.contextBudgetTokens! < preview.liveBudgetTokens;
|
|
2615
|
+
if (preview.path === 'immediate' && !e.fits && lowering) {
|
|
2616
|
+
const msg =
|
|
2617
|
+
`[budget-preflight] contextBudgetTokens=${patch.contextBudgetTokens} would NOT fit ` +
|
|
2618
|
+
`${agentName}: the folded floor renders ${e.finalTokens} tokens against hard budget ` +
|
|
2619
|
+
`${e.budgetTokens} (head=${e.headTokens ?? '?'} tail=${e.tailTokens ?? '?'} ` +
|
|
2620
|
+
`middle=${e.middleTokens ?? '?'}, deepest L${e.deepestLevel ?? '?'}` +
|
|
2621
|
+
`${e.exhausted ? ', picker exhausted' : ''}). Every compile at this budget would ` +
|
|
2622
|
+
`OverBudget-wedge the agent. Lower the floor first (drain compression quarantine / ` +
|
|
2623
|
+
`advance the merge ladder), use a paced descent (no \`immediate\`), or pass allowInfeasible.`;
|
|
2624
|
+
if (!opts?.allowInfeasible) throw new BudgetPreflightError(msg, preview);
|
|
2625
|
+
console.warn(`${msg} — applying anyway (allowInfeasible).`);
|
|
2626
|
+
} else if (preview.path === 'immediate' && !e.fits) {
|
|
2627
|
+
console.warn(
|
|
2628
|
+
`[budget-preflight] ${agentName} remains over the folded floor at ` +
|
|
2629
|
+
`${patch.contextBudgetTokens} (floor renders ${e.finalTokens}); this patch does ` +
|
|
2630
|
+
`not lower the budget, so applying — drain quarantine / advance merges to clear it.`,
|
|
2631
|
+
);
|
|
2632
|
+
} else if (preview.path === 'paced' && preview.advisory && !preview.advisory.fits) {
|
|
2633
|
+
console.warn(
|
|
2634
|
+
`[budget-preflight] paced descent for ${agentName} targets ` +
|
|
2635
|
+
`${preview.advisory.targetTokens} tokens, below the current folded floor — the ` +
|
|
2636
|
+
`transition will converge only as far as the floor allows (drain quarantine / ` +
|
|
2637
|
+
`advance merges to go lower). Applying; the live compile budget is unchanged.`,
|
|
2638
|
+
);
|
|
2639
|
+
}
|
|
2640
|
+
} else if (preview.reason) {
|
|
2641
|
+
console.warn(
|
|
2642
|
+
`[budget-preflight] preview unavailable for ${agentName} (${preview.reason}); ` +
|
|
2643
|
+
`applying without preflight`,
|
|
2644
|
+
);
|
|
2645
|
+
}
|
|
2646
|
+
}
|
|
2173
2647
|
const result = agent.updateRuntimeSettings(patch);
|
|
2174
2648
|
if (opts?.persist !== false) {
|
|
2175
2649
|
this.persistAgentRuntimeSettings(agentName, agent.getRuntimeSettingsOverrides());
|
|
2176
2650
|
}
|
|
2651
|
+
this.recordOperatorAction({
|
|
2652
|
+
kind: 'settings-update',
|
|
2653
|
+
agent: agentName,
|
|
2654
|
+
...(opts?.requester ? { requester: opts.requester } : {}),
|
|
2655
|
+
...(opts?.note ? { note: opts.note } : {}),
|
|
2656
|
+
params: { patch: { ...patch }, persist: opts?.persist !== false },
|
|
2657
|
+
});
|
|
2177
2658
|
return result;
|
|
2178
2659
|
}
|
|
2179
2660
|
|
|
2180
2661
|
resetAgentRuntimeSettings(
|
|
2181
2662
|
agentName: string,
|
|
2182
2663
|
keys?: Array<keyof AgentRuntimeSettingsPatch>,
|
|
2183
|
-
opts?: { persist?: boolean },
|
|
2664
|
+
opts?: { persist?: boolean; requester?: OperatorRequester; note?: string },
|
|
2184
2665
|
): AgentRuntimeSettingsSnapshot {
|
|
2185
2666
|
const agent = this.agents.get(agentName);
|
|
2186
2667
|
if (!agent) throw new Error(`Unknown agent: ${agentName}`);
|
|
@@ -2188,17 +2669,658 @@ export class AgentFramework {
|
|
|
2188
2669
|
if (opts?.persist !== false) {
|
|
2189
2670
|
this.persistAgentRuntimeSettings(agentName, agent.getRuntimeSettingsOverrides());
|
|
2190
2671
|
}
|
|
2672
|
+
this.recordOperatorAction({
|
|
2673
|
+
kind: 'settings-reset',
|
|
2674
|
+
agent: agentName,
|
|
2675
|
+
...(opts?.requester ? { requester: opts.requester } : {}),
|
|
2676
|
+
...(opts?.note ? { note: opts.note } : {}),
|
|
2677
|
+
params: { keys: keys ?? 'all', persist: opts?.persist !== false },
|
|
2678
|
+
});
|
|
2191
2679
|
return result;
|
|
2192
2680
|
}
|
|
2193
2681
|
|
|
2194
|
-
cancelAgentRuntimeSettingsTransition(
|
|
2682
|
+
cancelAgentRuntimeSettingsTransition(
|
|
2683
|
+
agentName: string,
|
|
2684
|
+
opts?: { requester?: OperatorRequester },
|
|
2685
|
+
): AgentRuntimeSettingsSnapshot {
|
|
2195
2686
|
const agent = this.agents.get(agentName);
|
|
2196
2687
|
if (!agent) throw new Error(`Unknown agent: ${agentName}`);
|
|
2197
2688
|
const result = agent.cancelRuntimeSettingsTransition();
|
|
2198
2689
|
this.persistAgentRuntimeSettings(agentName, agent.getRuntimeSettingsOverrides());
|
|
2690
|
+
this.recordOperatorAction({
|
|
2691
|
+
kind: 'settings-cancel-transition',
|
|
2692
|
+
agent: agentName,
|
|
2693
|
+
...(opts?.requester ? { requester: opts.requester } : {}),
|
|
2694
|
+
});
|
|
2199
2695
|
return result;
|
|
2200
2696
|
}
|
|
2201
2697
|
|
|
2698
|
+
// -------------------------------------------------------------------------
|
|
2699
|
+
// Host quiesce / maintenance mode (issue #122)
|
|
2700
|
+
//
|
|
2701
|
+
// Pause the inference thread and MCPL data planes while keeping the
|
|
2702
|
+
// framework + context managers + membrane loaded and hot, so maintenance
|
|
2703
|
+
// (compression ticks, refold, quarantine drains, budget descents) runs
|
|
2704
|
+
// through the exact machinery the agent uses live — with live config, live
|
|
2705
|
+
// tool definitions, and full llm-calls logging — instead of offline rigs
|
|
2706
|
+
// that re-derive all of it and drift.
|
|
2707
|
+
//
|
|
2708
|
+
// While quiesced: runLoop, store sync, the maintenance timer, and the
|
|
2709
|
+
// liveness watchdog all keep running. Wakes park in pendingRequests
|
|
2710
|
+
// (coalesced per reason) and fire at resume; MCPL events buffer on the
|
|
2711
|
+
// paused data planes; gate debounces buffer in the gate. Background
|
|
2712
|
+
// code-execution scripts are NOT stopped (they hold no turn token) — they
|
|
2713
|
+
// are reported in the status so the operator sees what still acts.
|
|
2714
|
+
//
|
|
2715
|
+
// NOTE: do not call quiesce() from inside a queue event handler — the drain
|
|
2716
|
+
// wait depends on the event loop continuing to run.
|
|
2717
|
+
// -------------------------------------------------------------------------
|
|
2718
|
+
|
|
2719
|
+
/**
|
|
2720
|
+
* Wakes that passed the scheduler and now own provider admission while
|
|
2721
|
+
* waiting for an in-flight auxiliary call to settle — no turn token yet,
|
|
2722
|
+
* so `activeTurnTokens` alone would call the host drained while a turn is
|
|
2723
|
+
* one promise-resolution away from starting. The admission continuation
|
|
2724
|
+
* rechecks quiesce and requeues, but until it runs the wake is live.
|
|
2725
|
+
*/
|
|
2726
|
+
private parkedAdmissionCount(): number {
|
|
2727
|
+
let n = 0;
|
|
2728
|
+
for (const [agentName, gate] of this.providerGates) {
|
|
2729
|
+
if (gate.primaryDepth > 0 && !this.activeTurnTokens.has(agentName)) n++;
|
|
2730
|
+
}
|
|
2731
|
+
return n;
|
|
2732
|
+
}
|
|
2733
|
+
|
|
2734
|
+
getHostModeStatus(): HostModeStatus {
|
|
2735
|
+
const parkedAdmissions = this.parkedAdmissionCount();
|
|
2736
|
+
return {
|
|
2737
|
+
quiesced: this.quiesced,
|
|
2738
|
+
...(this.quiesceReason ? { reason: this.quiesceReason } : {}),
|
|
2739
|
+
...(this.quiescedAt !== undefined ? { since: this.quiescedAt } : {}),
|
|
2740
|
+
drained: this.activeTurnTokens.size === 0 && parkedAdmissions === 0,
|
|
2741
|
+
activeTurns: this.activeTurnTokens.size,
|
|
2742
|
+
parkedAdmissions,
|
|
2743
|
+
gatedRequests: this.pendingRequests.filter((r) => !isTurnContinuation(r.reason)).length,
|
|
2744
|
+
backgroundScripts: [...this.backgroundScripts.values()]
|
|
2745
|
+
.filter((record) => record.status === 'running').length,
|
|
2746
|
+
deferredWrites: this.deferredMessages.length,
|
|
2747
|
+
...(this.lastUnabandonable.length > 0 ? { unabandonable: [...this.lastUnabandonable] } : {}),
|
|
2748
|
+
};
|
|
2749
|
+
}
|
|
2750
|
+
|
|
2751
|
+
/** Drain window bounds shared by every ingress (framework API, WS, HTTP,
|
|
2752
|
+
* host/command). Below the floor a drain is meaningless; above the ceiling
|
|
2753
|
+
* an awaited call would poll for days. */
|
|
2754
|
+
static readonly QUIESCE_TIMEOUT_MIN_MS = 1_000;
|
|
2755
|
+
static readonly QUIESCE_TIMEOUT_MAX_MS = 600_000;
|
|
2756
|
+
static readonly QUIESCE_TIMEOUT_DEFAULT_MS = 120_000;
|
|
2757
|
+
|
|
2758
|
+
/**
|
|
2759
|
+
* Enter quiesce mode: persist the flag, suppress gate deliveries, pause all
|
|
2760
|
+
* MCPL data planes, and wait for in-flight turns to settle (turn-alive is
|
|
2761
|
+
* `activeTurnTokens`, which spans dequeue → settled teardown). Idempotent.
|
|
2762
|
+
*
|
|
2763
|
+
* On drain timeout the host STAYS quiesced (`drained: false` in the
|
|
2764
|
+
* result); with `abandon: true` the undrained streams are cancelled via a
|
|
2765
|
+
* dedicated cancel kind that settles the turn without feeding the
|
|
2766
|
+
* inference-failure accounting.
|
|
2767
|
+
*/
|
|
2768
|
+
async quiesce(opts?: {
|
|
2769
|
+
reason?: string;
|
|
2770
|
+
timeoutMs?: number;
|
|
2771
|
+
abandon?: boolean;
|
|
2772
|
+
}): Promise<HostModeStatus> {
|
|
2773
|
+
if (this.quiesced) {
|
|
2774
|
+
// Idempotent — EXCEPT abandon escalation: a first quiesce that timed
|
|
2775
|
+
// out undrained must be escalatable with a second quiesce({abandon})
|
|
2776
|
+
// without resume()+re-quiesce (which would reopen data planes and
|
|
2777
|
+
// release parked wakes mid-surgery).
|
|
2778
|
+
if (opts?.abandon && this.activeTurnTokens.size > 0) {
|
|
2779
|
+
const unabandonable = await this.abandonActiveTurns();
|
|
2780
|
+
this.emitTrace({
|
|
2781
|
+
type: 'host:quiesce',
|
|
2782
|
+
...(this.quiesceReason ? { reason: this.quiesceReason } : {}),
|
|
2783
|
+
drained: this.activeTurnTokens.size === 0,
|
|
2784
|
+
activeTurns: this.activeTurnTokens.size,
|
|
2785
|
+
abandoned: true,
|
|
2786
|
+
...(unabandonable.length > 0 ? { unabandonable } : {}),
|
|
2787
|
+
});
|
|
2788
|
+
}
|
|
2789
|
+
return this.getHostModeStatus();
|
|
2790
|
+
}
|
|
2791
|
+
this.quiesced = true;
|
|
2792
|
+
this.quiesceReason = opts?.reason;
|
|
2793
|
+
this.quiescedAt = Date.now();
|
|
2794
|
+
this.lastUnabandonable = [];
|
|
2795
|
+
this.persistHostMode({
|
|
2796
|
+
quiesced: true,
|
|
2797
|
+
...(opts?.reason ? { reason: opts.reason } : {}),
|
|
2798
|
+
since: this.quiescedAt,
|
|
2799
|
+
});
|
|
2800
|
+
console.error(
|
|
2801
|
+
`[host-mode] quiescing${opts?.reason ? ` (${opts.reason})` : ''}: parking wakes, ` +
|
|
2802
|
+
`pausing MCPL data planes, draining ${this.activeTurnTokens.size} in-flight turn(s)`,
|
|
2803
|
+
);
|
|
2804
|
+
this.eventGate?.setQuiesced(true);
|
|
2805
|
+
for (const connection of this.mcplServerRegistry?.getAllServers() ?? []) {
|
|
2806
|
+
connection.pauseDataPlane();
|
|
2807
|
+
}
|
|
2808
|
+
|
|
2809
|
+
// Clamp HERE, for every ingress. NaN-proof too: timeoutMs arrives via
|
|
2810
|
+
// Number()/blind casts, and Math.max(1_000, NaN) is NaN — which would
|
|
2811
|
+
// silently collapse the drain window to zero (`Date.now() < NaN` is
|
|
2812
|
+
// false). The ceiling matters because this call is awaited: an
|
|
2813
|
+
// unbounded value polls the event loop for as long as it says.
|
|
2814
|
+
const rawTimeout = opts?.timeoutMs;
|
|
2815
|
+
const timeoutMs = typeof rawTimeout === 'number' && Number.isFinite(rawTimeout)
|
|
2816
|
+
? Math.max(
|
|
2817
|
+
AgentFramework.QUIESCE_TIMEOUT_MIN_MS,
|
|
2818
|
+
Math.min(AgentFramework.QUIESCE_TIMEOUT_MAX_MS, Math.floor(rawTimeout)),
|
|
2819
|
+
)
|
|
2820
|
+
: AgentFramework.QUIESCE_TIMEOUT_DEFAULT_MS;
|
|
2821
|
+
const deadline = Date.now() + timeoutMs;
|
|
2822
|
+
// Drain = no turn token AND no wake parked on provider admission (the
|
|
2823
|
+
// latter becomes a turn the moment its auxiliary settles unless the
|
|
2824
|
+
// continuation's quiesce recheck requeues it — wait for that to happen).
|
|
2825
|
+
while (
|
|
2826
|
+
this.quiesced &&
|
|
2827
|
+
(this.activeTurnTokens.size > 0 || this.parkedAdmissionCount() > 0) &&
|
|
2828
|
+
Date.now() < deadline
|
|
2829
|
+
) {
|
|
2830
|
+
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
2831
|
+
}
|
|
2832
|
+
|
|
2833
|
+
// A concurrent resume() ended this window while we waited. Nothing here
|
|
2834
|
+
// may act any more — an abandon now would cancel a turn that started
|
|
2835
|
+
// legitimately AFTER the resume, and the trace/log would describe a mode
|
|
2836
|
+
// the host is no longer in.
|
|
2837
|
+
if (!this.quiesced) {
|
|
2838
|
+
console.error('[host-mode] quiesce superseded by a concurrent resume during drain');
|
|
2839
|
+
return this.getHostModeStatus();
|
|
2840
|
+
}
|
|
2841
|
+
|
|
2842
|
+
let abandoned = false;
|
|
2843
|
+
let unabandonable: string[] = [];
|
|
2844
|
+
if (this.activeTurnTokens.size > 0 && opts?.abandon) {
|
|
2845
|
+
abandoned = true;
|
|
2846
|
+
unabandonable = await this.abandonActiveTurns();
|
|
2847
|
+
if (!this.quiesced) return this.getHostModeStatus();
|
|
2848
|
+
}
|
|
2849
|
+
|
|
2850
|
+
const status = this.getHostModeStatus();
|
|
2851
|
+
this.emitTrace({
|
|
2852
|
+
type: 'host:quiesce',
|
|
2853
|
+
...(this.quiesceReason ? { reason: this.quiesceReason } : {}),
|
|
2854
|
+
drained: status.drained,
|
|
2855
|
+
activeTurns: status.activeTurns,
|
|
2856
|
+
...(status.parkedAdmissions > 0 ? { parkedAdmissions: status.parkedAdmissions } : {}),
|
|
2857
|
+
...(abandoned ? { abandoned: true } : {}),
|
|
2858
|
+
...(unabandonable.length > 0 ? { unabandonable } : {}),
|
|
2859
|
+
});
|
|
2860
|
+
console.error(
|
|
2861
|
+
status.drained
|
|
2862
|
+
? `[host-mode] quiesced — drained, ${status.gatedRequests} wake(s) parked`
|
|
2863
|
+
: `[host-mode] quiesced but drain ${abandoned ? 'needed abandon and' : 'timed out —'} ` +
|
|
2864
|
+
`${status.activeTurns} turn(s) still alive` +
|
|
2865
|
+
(status.parkedAdmissions > 0
|
|
2866
|
+
? `, ${status.parkedAdmissions} wake(s) parked on provider admission`
|
|
2867
|
+
: ''),
|
|
2868
|
+
);
|
|
2869
|
+
return status;
|
|
2870
|
+
}
|
|
2871
|
+
|
|
2872
|
+
/** Cancel every turn-alive stream via the quiesce_abandoned kind and give
|
|
2873
|
+
* teardown a bounded grace to settle. Shared by the drain-timeout path and
|
|
2874
|
+
* the already-quiesced escalation path. */
|
|
2875
|
+
private async abandonActiveTurns(): Promise<string[]> {
|
|
2876
|
+
const unabandonable: string[] = [];
|
|
2877
|
+
for (const agentName of [...this.activeTurnTokens.keys()]) {
|
|
2878
|
+
const agent = this.agents.get(agentName);
|
|
2879
|
+
const state = agent?.state;
|
|
2880
|
+
const stream = state && 'stream' in state ? state.stream : undefined;
|
|
2881
|
+
if (agent && stream) {
|
|
2882
|
+
console.error(`[host-mode] abandoning in-flight turn for ${agentName}`);
|
|
2883
|
+
this.frameworkCancelledStreams.set(
|
|
2884
|
+
`${agent.name}:${agent.streamId}`,
|
|
2885
|
+
'quiesce_abandoned',
|
|
2886
|
+
);
|
|
2887
|
+
stream.cancel();
|
|
2888
|
+
} else {
|
|
2889
|
+
// Token held, no stream to cancel: the turn is between dequeue and
|
|
2890
|
+
// stream registration (hooks/compile), or it is a puppet tool turn,
|
|
2891
|
+
// which never has one. Say so — silence here reads as "abandon acted
|
|
2892
|
+
// and teardown is slow", which is a different operator decision.
|
|
2893
|
+
unabandonable.push(agentName);
|
|
2894
|
+
console.error(
|
|
2895
|
+
`[host-mode] cannot abandon turn for ${agentName}: turn token held with no stream ` +
|
|
2896
|
+
`(status=${state?.status ?? 'unknown'}) — it must settle on its own`,
|
|
2897
|
+
);
|
|
2898
|
+
}
|
|
2899
|
+
}
|
|
2900
|
+
// Bounded grace for the cancelled streams' teardown to settle.
|
|
2901
|
+
const grace = Date.now() + 10_000;
|
|
2902
|
+
while (this.quiesced && this.activeTurnTokens.size > 0 && Date.now() < grace) {
|
|
2903
|
+
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
2904
|
+
}
|
|
2905
|
+
// Only turns still alive after the grace are worth reporting.
|
|
2906
|
+
this.lastUnabandonable = unabandonable.filter((name) => this.activeTurnTokens.has(name));
|
|
2907
|
+
return this.lastUnabandonable;
|
|
2908
|
+
}
|
|
2909
|
+
|
|
2910
|
+
/**
|
|
2911
|
+
* Leave quiesce mode. Gates on a FRESH feasibility preview of every agent's
|
|
2912
|
+
* CURRENT settings — maintenance may have moved the folded floor and the
|
|
2913
|
+
* branch generation, so verdicts are computed at resume time, never reused.
|
|
2914
|
+
* Throws ResumeBlockedError (with all failing verdicts) unless `force`.
|
|
2915
|
+
* Unavailable previews warn and pass — refusing to resume because the
|
|
2916
|
+
* strategy cannot preview would hold hosts hostage to a diagnostic.
|
|
2917
|
+
*/
|
|
2918
|
+
async resume(opts?: { force?: boolean }): Promise<HostModeStatus> {
|
|
2919
|
+
if (!this.quiesced) return this.getHostModeStatus();
|
|
2920
|
+
|
|
2921
|
+
// The canonical operator flow is `maintain; resume`, and a timer-driven
|
|
2922
|
+
// pass may still be mid-flight: previewContext refuses to overlap one
|
|
2923
|
+
// ("already running"), which used to degrade the gate to "no opinion".
|
|
2924
|
+
// Wait for the pass first, then retry a few times for the case where a
|
|
2925
|
+
// fresh pass started in between.
|
|
2926
|
+
if (this.maintenancePass) {
|
|
2927
|
+
await this.maintenancePass.catch(() => {});
|
|
2928
|
+
}
|
|
2929
|
+
const failing: Array<{ agentName: string; preview: RuntimeSettingsPreview }> = [];
|
|
2930
|
+
for (const agentName of this.agents.keys()) {
|
|
2931
|
+
let preview = this.previewAgentRuntimeSettings(agentName);
|
|
2932
|
+
for (let attempt = 0; attempt < 8 && !preview.available && preview.reason === 'preview_in_flight'; attempt++) {
|
|
2933
|
+
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
2934
|
+
if (this.maintenancePass) await this.maintenancePass.catch(() => {});
|
|
2935
|
+
preview = this.previewAgentRuntimeSettings(agentName);
|
|
2936
|
+
}
|
|
2937
|
+
if (preview.available && preview.effective && !preview.effective.fits) {
|
|
2938
|
+
failing.push({ agentName, preview });
|
|
2939
|
+
} else if (!preview.available && preview.reason !== 'no_preview_support') {
|
|
2940
|
+
console.warn(
|
|
2941
|
+
`[host-mode] resume: feasibility preview unavailable for ${agentName} ` +
|
|
2942
|
+
`(${preview.reason}) — proceeding without a verdict`,
|
|
2943
|
+
);
|
|
2944
|
+
}
|
|
2945
|
+
}
|
|
2946
|
+
if (failing.length > 0 && !opts?.force) {
|
|
2947
|
+
const detail = failing.map(({ agentName, preview }) =>
|
|
2948
|
+
`${agentName}: folded floor ${preview.effective!.finalTokens} > hard budget ` +
|
|
2949
|
+
`${preview.effective!.budgetTokens}` +
|
|
2950
|
+
(preview.transition === 'blocked'
|
|
2951
|
+
? ` (transition blocked: ${preview.transitionReason ?? 'unknown'})`
|
|
2952
|
+
: ''),
|
|
2953
|
+
).join('; ');
|
|
2954
|
+
throw new ResumeBlockedError(
|
|
2955
|
+
`resume refused: returning to service would OverBudget-wedge — ${detail}. ` +
|
|
2956
|
+
`Drain compression quarantine / advance the merge ladder to lower the floor ` +
|
|
2957
|
+
`(maintenanceTick()), or resume({ force: true }).`,
|
|
2958
|
+
failing,
|
|
2959
|
+
);
|
|
2960
|
+
}
|
|
2961
|
+
if (failing.length > 0) {
|
|
2962
|
+
console.warn(
|
|
2963
|
+
`[host-mode] resume FORCED past ${failing.length} failing feasibility verdict(s)`,
|
|
2964
|
+
);
|
|
2965
|
+
}
|
|
2966
|
+
|
|
2967
|
+
const releasedRequests = this.pendingRequests.length;
|
|
2968
|
+
// In-memory first: addMessage must stop deferring so the flush below can
|
|
2969
|
+
// land. The DURABLE flag is cleared only after the flush completes — a
|
|
2970
|
+
// crash in between then boots quiesced with the un-acked remainder
|
|
2971
|
+
// restored (and a serving boot recovers any remainder anyway).
|
|
2972
|
+
this.quiesced = false;
|
|
2973
|
+
this.quiesceReason = undefined;
|
|
2974
|
+
this.quiescedAt = undefined;
|
|
2975
|
+
this.lastUnabandonable = [];
|
|
2976
|
+
this.eventGate?.setQuiesced(false);
|
|
2977
|
+
|
|
2978
|
+
try {
|
|
2979
|
+
// Flush context writes deferred by the quiesce window (module events,
|
|
2980
|
+
// api message.send) — with no turn alive there is no other flush
|
|
2981
|
+
// point. Per-message try/catch and per-message durable ack (see
|
|
2982
|
+
// flushDeferredWrites); nothing here may reach the plane reopen below.
|
|
2983
|
+
await this.flushDeferredWrites('resume');
|
|
2984
|
+
this.persistHostMode(null);
|
|
2985
|
+
} finally {
|
|
2986
|
+
// Reopen MCPL data planes through the existing barrier funnel — NOT a
|
|
2987
|
+
// bespoke ready() loop. The funnel inherits completeMcplDataPlaneGate's
|
|
2988
|
+
// nested-install guard (a flushed tools-list-changed can install a
|
|
2989
|
+
// newer barrier mid-flush), drains any awareness work accumulated
|
|
2990
|
+
// during the window before opening, and replaces a stale failed
|
|
2991
|
+
// barrier by identity. In `finally` so that nothing above can leave
|
|
2992
|
+
// the host un-quiesced with every data plane still paused.
|
|
2993
|
+
if (this.mcplServerRegistry) {
|
|
2994
|
+
const barrier = this.installMcplDataPlaneGate();
|
|
2995
|
+
this.releaseMcplDataPlaneGate(barrier);
|
|
2996
|
+
try {
|
|
2997
|
+
await barrier.promise;
|
|
2998
|
+
this.completeMcplDataPlaneGate(barrier);
|
|
2999
|
+
} catch (error) {
|
|
3000
|
+
// The host IS resumed — don't rethrow. failMcplDataPlaneGate
|
|
3001
|
+
// recycles the connections, and their reconnect flows re-run the
|
|
3002
|
+
// funnel with quiesced=false, self-healing the data planes.
|
|
3003
|
+
await this.failMcplDataPlaneGate(barrier, 'quiesce resume', error);
|
|
3004
|
+
}
|
|
3005
|
+
}
|
|
3006
|
+
|
|
3007
|
+
this.emitTrace({
|
|
3008
|
+
type: 'host:resume',
|
|
3009
|
+
...(opts?.force && failing.length > 0 ? { forced: true } : {}),
|
|
3010
|
+
releasedRequests,
|
|
3011
|
+
});
|
|
3012
|
+
// NOTE: parked wakes and gate buffers are in-memory only — a restart
|
|
3013
|
+
// mid-quiesce boots quiesced but with these queues empty (sources like
|
|
3014
|
+
// heartbeats re-deliver on their own cadence; one-shot wakes are lost).
|
|
3015
|
+
// Deferred context WRITES are persisted while quiesced and survive.
|
|
3016
|
+
console.error(`[host-mode] resumed — ${releasedRequests} parked wake(s) released`);
|
|
3017
|
+
}
|
|
3018
|
+
return this.getHostModeStatus();
|
|
3019
|
+
}
|
|
3020
|
+
|
|
3021
|
+
/**
|
|
3022
|
+
* Land every deferred write whose target has no turn alive, one at a time,
|
|
3023
|
+
* acknowledging each durably as it lands (the persisted queue is rewritten
|
|
3024
|
+
* after every message). A crash mid-flush therefore replays exactly the
|
|
3025
|
+
* un-acked remainder at the next boot — no loss, no duplicates. A write
|
|
3026
|
+
* the store REJECTS is logged and acked too: replaying a poison message
|
|
3027
|
+
* forever would wedge every later boot on it.
|
|
3028
|
+
*
|
|
3029
|
+
* Per-target: an agent whose turn is still alive keeps ITS messages
|
|
3030
|
+
* deferred for its own turn boundary; everyone else's flush now.
|
|
3031
|
+
*/
|
|
3032
|
+
private async flushDeferredWrites(label: string): Promise<void> {
|
|
3033
|
+
if (this.deferredMessages.length === 0) {
|
|
3034
|
+
this.persistDeferredWrites();
|
|
3035
|
+
return;
|
|
3036
|
+
}
|
|
3037
|
+
const keep: typeof this.deferredMessages = [];
|
|
3038
|
+
const flush: typeof this.deferredMessages = [];
|
|
3039
|
+
for (const msg of this.deferredMessages) {
|
|
3040
|
+
const target = msg.forAgent ?? this.primaryAgentName;
|
|
3041
|
+
(target && this.activeTurnTokens.has(target) ? keep : flush).push(msg);
|
|
3042
|
+
}
|
|
3043
|
+
// Hand-off: the batch leaves the pending queue and enters the un-acked
|
|
3044
|
+
// set. The DURABLE queue (pending + un-acked) is unchanged by this, so a
|
|
3045
|
+
// crash anywhere below replays the whole batch, and the deferredWriteId
|
|
3046
|
+
// stamped on each stored message lets boot skip the ones that did land.
|
|
3047
|
+
flush.sort(bySeq);
|
|
3048
|
+
this.deferredMessages = keep;
|
|
3049
|
+
this.handOffDeferredWrites(flush);
|
|
3050
|
+
let stored = 0;
|
|
3051
|
+
for (const msg of flush) {
|
|
3052
|
+
try {
|
|
3053
|
+
this.addMessage(msg.participant, msg.content, msg.metadata, {
|
|
3054
|
+
deferredWriteId: msg.id,
|
|
3055
|
+
...(msg.forAgent ? { forAgent: msg.forAgent } : {}),
|
|
3056
|
+
});
|
|
3057
|
+
stored++;
|
|
3058
|
+
} catch (err) {
|
|
3059
|
+
console.error(
|
|
3060
|
+
`[host-mode] ${label}: failed to store a deferred context write ` +
|
|
3061
|
+
`(participant=${msg.participant}, forAgent=${msg.forAgent ?? 'primary'}):`,
|
|
3062
|
+
err,
|
|
3063
|
+
);
|
|
3064
|
+
}
|
|
3065
|
+
}
|
|
3066
|
+
// Ack = sync the chronicle FIRST, then rewrite the durable queue without
|
|
3067
|
+
// the batch. Never the other way round.
|
|
3068
|
+
this.ackDeferredWrites();
|
|
3069
|
+
console.error(
|
|
3070
|
+
`[host-mode] ${label}: flushed ${stored}/${flush.length} deferred context write(s)` +
|
|
3071
|
+
(keep.length > 0 ? `, ${keep.length} kept for a still-alive turn` : ''),
|
|
3072
|
+
);
|
|
3073
|
+
}
|
|
3074
|
+
|
|
3075
|
+
/**
|
|
3076
|
+
* Acknowledge every deferred write that has been handed to a context
|
|
3077
|
+
* manager: sync the chronicle so the appended slots are durable, and only
|
|
3078
|
+
* then drop them from the recovery queue. If the sync fails they stay in
|
|
3079
|
+
* the durable queue (the next ack, or stop(), retries); a reboot in that
|
|
3080
|
+
* state replays them, deduplicated by `deferredWriteId` for any that did
|
|
3081
|
+
* reach disk. Idempotent and cheap when nothing is un-acked.
|
|
3082
|
+
*/
|
|
3083
|
+
private ackDeferredWrites(): void {
|
|
3084
|
+
if (this.unackedDeferredWrites.length === 0) return;
|
|
3085
|
+
// Nothing durable to reconcile against unless the queue was persisted.
|
|
3086
|
+
if (!this.deferredWritesPersisted && !this.quiesced) {
|
|
3087
|
+
this.unackedDeferredWrites = [];
|
|
3088
|
+
return;
|
|
3089
|
+
}
|
|
3090
|
+
try {
|
|
3091
|
+
this.store.sync();
|
|
3092
|
+
} catch (err) {
|
|
3093
|
+
console.error(
|
|
3094
|
+
`[host-mode] chronicle sync failed — ${this.unackedDeferredWrites.length} flushed deferred ` +
|
|
3095
|
+
`write(s) stay in the recovery queue until a sync succeeds:`,
|
|
3096
|
+
err,
|
|
3097
|
+
);
|
|
3098
|
+
return;
|
|
3099
|
+
}
|
|
3100
|
+
this.unackedDeferredWrites = [];
|
|
3101
|
+
this.deferredScanFrom.clear();
|
|
3102
|
+
this.persistDeferredWrites();
|
|
3103
|
+
}
|
|
3104
|
+
|
|
3105
|
+
/**
|
|
3106
|
+
* Move drained entries into the un-acked set and record, per target
|
|
3107
|
+
* agent, the store position they will be appended after — then persist
|
|
3108
|
+
* that receipt BEFORE any of them is written. Boot recovery scans from the
|
|
3109
|
+
* recorded position to the tail, so every member of an interrupted batch
|
|
3110
|
+
* is visible to the dedup, however large the batch.
|
|
3111
|
+
*/
|
|
3112
|
+
private handOffDeferredWrites(entries: DeferredWrite[]): void {
|
|
3113
|
+
if (entries.length === 0) return;
|
|
3114
|
+
const durable = this.deferredWritesPersisted || this.quiesced;
|
|
3115
|
+
if (!durable) return; // memory-only deferrals: nothing to reconcile at boot
|
|
3116
|
+
for (const entry of entries) {
|
|
3117
|
+
const target = entry.forAgent ?? this.primaryAgentName;
|
|
3118
|
+
const agent = target ? this.agents.get(target) : undefined;
|
|
3119
|
+
if (!agent) continue;
|
|
3120
|
+
let count = 0;
|
|
3121
|
+
try {
|
|
3122
|
+
const cm = agent.getContextManager() as unknown as { getMessageCount?: () => number; getAllMessages: () => unknown[] };
|
|
3123
|
+
count = typeof cm.getMessageCount === 'function' ? cm.getMessageCount() : cm.getAllMessages().length;
|
|
3124
|
+
} catch { count = 0; }
|
|
3125
|
+
const prev = this.deferredScanFrom.get(target!);
|
|
3126
|
+
this.deferredScanFrom.set(target!, prev === undefined ? count : Math.min(prev, count));
|
|
3127
|
+
}
|
|
3128
|
+
this.unackedDeferredWrites.push(...entries);
|
|
3129
|
+
this.persistDeferredWrites();
|
|
3130
|
+
}
|
|
3131
|
+
|
|
3132
|
+
/**
|
|
3133
|
+
* Which of `ids` already exist in some agent's store as a landed deferred
|
|
3134
|
+
* write (`metadata.deferredWriteId`). Scans each context manager's recent
|
|
3135
|
+
* tail blob-free — an interrupted flush is always within the last few
|
|
3136
|
+
* hundred slots — and tolerates facades without windowed reads.
|
|
3137
|
+
*/
|
|
3138
|
+
private landedDeferredWriteIds(ids: Set<string>, scanFrom: Map<string, number>): Set<string> {
|
|
3139
|
+
const landed = new Set<string>();
|
|
3140
|
+
if (ids.size === 0) return landed;
|
|
3141
|
+
const WINDOW = 500;
|
|
3142
|
+
for (const agent of this.agents.values()) {
|
|
3143
|
+
try {
|
|
3144
|
+
const cm = agent.getContextManager() as unknown as {
|
|
3145
|
+
getMessageCount?: () => number;
|
|
3146
|
+
getMessageWindow?: (o: number, l: number, opts?: { resolveBlobs?: boolean }) =>
|
|
3147
|
+
{ messages: Array<{ metadata?: Record<string, unknown> }> };
|
|
3148
|
+
getAllMessages: () => Array<{ metadata?: Record<string, unknown> }>;
|
|
3149
|
+
};
|
|
3150
|
+
const consider = (m: { metadata?: Record<string, unknown> }): void => {
|
|
3151
|
+
const id = m.metadata?.deferredWriteId;
|
|
3152
|
+
if (typeof id === 'string' && ids.has(id)) landed.add(id);
|
|
3153
|
+
};
|
|
3154
|
+
if (typeof cm.getMessageCount === 'function' && typeof cm.getMessageWindow === 'function') {
|
|
3155
|
+
const total = cm.getMessageCount();
|
|
3156
|
+
// From the receipt position (0 = whole store when unknown) to the
|
|
3157
|
+
// tail, in blob-free windows — the batch can only be after it.
|
|
3158
|
+
const from = Math.min(scanFrom.get(agent.name) ?? 0, total);
|
|
3159
|
+
for (let start = from; start < total; start += WINDOW) {
|
|
3160
|
+
const win = cm.getMessageWindow(start, Math.min(WINDOW, total - start), { resolveBlobs: false });
|
|
3161
|
+
for (const m of win.messages) consider(m);
|
|
3162
|
+
}
|
|
3163
|
+
} else {
|
|
3164
|
+
for (const m of cm.getAllMessages()) consider(m);
|
|
3165
|
+
}
|
|
3166
|
+
} catch (err) {
|
|
3167
|
+
console.error(`[host-mode] could not scan ${agent.name} for landed deferred writes:`, err);
|
|
3168
|
+
}
|
|
3169
|
+
}
|
|
3170
|
+
return landed;
|
|
3171
|
+
}
|
|
3172
|
+
|
|
3173
|
+
/** Atomic JSON write for the recovery files (tmp + rename, 0600) — the
|
|
3174
|
+
* same idiom as the Discord awareness outbox. */
|
|
3175
|
+
private writeRecoveryFile(path: string, document: unknown): void {
|
|
3176
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
3177
|
+
const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
|
|
3178
|
+
writeFileSync(temporary, `${JSON.stringify(document, null, 2)}\n`, { mode: 0o600 });
|
|
3179
|
+
renameSync(temporary, path);
|
|
3180
|
+
}
|
|
3181
|
+
|
|
3182
|
+
private readRecoveryFile(path: string): unknown {
|
|
3183
|
+
if (!existsSync(path)) return undefined;
|
|
3184
|
+
return JSON.parse(readFileSync(path, 'utf8'));
|
|
3185
|
+
}
|
|
3186
|
+
|
|
3187
|
+
private warnRecoveryFallbackOnce(): void {
|
|
3188
|
+
if (this.recoveryFallbackWarned) return;
|
|
3189
|
+
this.recoveryFallbackWarned = true;
|
|
3190
|
+
console.error(
|
|
3191
|
+
'[host-mode] no storePath / hostModePath / deferredWritesPath: quiesce state and deferred ' +
|
|
3192
|
+
'writes persist in the branch-local framework/state slot — a historical rollback while ' +
|
|
3193
|
+
'quiesced can erase them; configure the recovery paths',
|
|
3194
|
+
);
|
|
3195
|
+
}
|
|
3196
|
+
|
|
3197
|
+
/**
|
|
3198
|
+
* Mirror the deferred-write queue durably while quiesced (and until an
|
|
3199
|
+
* already-persisted queue drains). The quiesce flag itself is persisted
|
|
3200
|
+
* because a crash mid-surgery is an expected event; the messages withheld
|
|
3201
|
+
* BECAUSE of that surgery would otherwise die with the process — and the
|
|
3202
|
+
* window is operator-length, not turn-length. Lives in the
|
|
3203
|
+
* branch-independent recovery file so a rollback cannot orphan it.
|
|
3204
|
+
* Deferrals outside a quiesce window (turn-alive, mid-tool-cycle) are
|
|
3205
|
+
* still memory-only: they flush within the turn, as before.
|
|
3206
|
+
*/
|
|
3207
|
+
private persistDeferredWrites(): void {
|
|
3208
|
+
try {
|
|
3209
|
+
// The durable queue is everything not yet acked: writes still pending
|
|
3210
|
+
// AND writes handed to a context manager whose sync has not happened.
|
|
3211
|
+
// Always in original deferral order, whatever the pending/un-acked
|
|
3212
|
+
// split: a re-deferred entry must not jump the queue.
|
|
3213
|
+
const durable = [...this.unackedDeferredWrites, ...this.deferredMessages].sort(bySeq);
|
|
3214
|
+
const scanFrom = Object.fromEntries(this.deferredScanFrom);
|
|
3215
|
+
if ((this.quiesced || this.deferredWritesPersisted) && durable.length > 0) {
|
|
3216
|
+
const payload = JSON.stringify(durable);
|
|
3217
|
+
if (payload.length > DEFERRED_WRITES_PERSIST_CAP_BYTES) {
|
|
3218
|
+
if (!this.deferredWritesCapWarned) {
|
|
3219
|
+
this.deferredWritesCapWarned = true;
|
|
3220
|
+
console.error(
|
|
3221
|
+
`[host-mode] deferred context writes exceed ${DEFERRED_WRITES_PERSIST_CAP_BYTES} bytes ` +
|
|
3222
|
+
`serialized (${durable.length} message(s)) — kept in memory only; ` +
|
|
3223
|
+
`a crash before resume loses them`,
|
|
3224
|
+
);
|
|
3225
|
+
}
|
|
3226
|
+
return;
|
|
3227
|
+
}
|
|
3228
|
+
if (this.deferredWritesPath) {
|
|
3229
|
+
this.writeRecoveryFile(this.deferredWritesPath, { version: 2, pending: durable, scanFrom });
|
|
3230
|
+
} else {
|
|
3231
|
+
this.warnRecoveryFallbackOnce();
|
|
3232
|
+
this.store.setStateJson(DEFERRED_WRITES_ID, { version: 2, pending: durable, scanFrom });
|
|
3233
|
+
}
|
|
3234
|
+
this.deferredWritesPersisted = true;
|
|
3235
|
+
} else if (this.deferredWritesPersisted) {
|
|
3236
|
+
if (this.deferredWritesPath) {
|
|
3237
|
+
this.writeRecoveryFile(this.deferredWritesPath, { version: 1, pending: [] });
|
|
3238
|
+
} else {
|
|
3239
|
+
this.store.setStateJson(DEFERRED_WRITES_ID, []);
|
|
3240
|
+
}
|
|
3241
|
+
this.deferredWritesPersisted = false;
|
|
3242
|
+
}
|
|
3243
|
+
} catch (err) {
|
|
3244
|
+
console.error('[host-mode] failed to persist deferred context writes:', err);
|
|
3245
|
+
}
|
|
3246
|
+
}
|
|
3247
|
+
|
|
3248
|
+
private restorePersistedDeferredWrites(): number {
|
|
3249
|
+
try {
|
|
3250
|
+
let raw: unknown;
|
|
3251
|
+
if (this.deferredWritesPath && existsSync(this.deferredWritesPath)) {
|
|
3252
|
+
raw = this.readRecoveryFile(this.deferredWritesPath);
|
|
3253
|
+
} else {
|
|
3254
|
+
// Stores that predate the recovery file (or store-only configs).
|
|
3255
|
+
raw = this.store.getStateJson(DEFERRED_WRITES_ID);
|
|
3256
|
+
}
|
|
3257
|
+
// v1 slot payloads were a bare array; v1/v2 files and v2 slots are
|
|
3258
|
+
// `{ pending, scanFrom? }`.
|
|
3259
|
+
const doc = Array.isArray(raw) ? { pending: raw } : (raw as { pending?: unknown; scanFrom?: unknown } | undefined);
|
|
3260
|
+
const data = doc?.pending;
|
|
3261
|
+
if (!Array.isArray(data) || data.length === 0) return 0;
|
|
3262
|
+
const scanFrom = new Map<string, number>();
|
|
3263
|
+
if (doc?.scanFrom && typeof doc.scanFrom === 'object') {
|
|
3264
|
+
for (const [k, v] of Object.entries(doc.scanFrom as Record<string, unknown>)) {
|
|
3265
|
+
if (typeof v === 'number' && Number.isFinite(v)) scanFrom.set(k, Math.max(0, Math.floor(v)));
|
|
3266
|
+
}
|
|
3267
|
+
}
|
|
3268
|
+
const candidates = data
|
|
3269
|
+
.filter((m): m is Omit<DeferredWrite, 'id' | 'seq'> & { id?: string; seq?: number } =>
|
|
3270
|
+
!!m && typeof m === 'object' && typeof (m as { participant?: unknown }).participant === 'string'
|
|
3271
|
+
&& Array.isArray((m as { content?: unknown }).content))
|
|
3272
|
+
.map((m, i) => ({
|
|
3273
|
+
...m,
|
|
3274
|
+
id: typeof m.id === 'string' ? m.id : randomUUID(),
|
|
3275
|
+
// Files that predate `seq` are in write order already.
|
|
3276
|
+
seq: typeof m.seq === 'number' ? m.seq : i + 1,
|
|
3277
|
+
}))
|
|
3278
|
+
.sort(bySeq);
|
|
3279
|
+
this.deferredSeq = Math.max(this.deferredSeq, ...candidates.map((m) => m.seq));
|
|
3280
|
+
// Exactly-once: a message that landed before its ack was written is
|
|
3281
|
+
// already in the store under its deferredWriteId — do not replay it.
|
|
3282
|
+
// The scan covers every slot the interrupted batch could occupy (from
|
|
3283
|
+
// the receipt's per-agent position), or the whole store when the
|
|
3284
|
+
// receipt predates the position field.
|
|
3285
|
+
const landed = this.landedDeferredWriteIds(new Set(candidates.map((m) => m.id)), scanFrom);
|
|
3286
|
+
const restored = candidates.filter((m) => !landed.has(m.id));
|
|
3287
|
+
if (landed.size > 0) {
|
|
3288
|
+
console.error(
|
|
3289
|
+
`[host-mode] ${landed.size} deferred context write(s) had already landed before their ack ` +
|
|
3290
|
+
`(interrupted flush) — skipped, not replayed`,
|
|
3291
|
+
);
|
|
3292
|
+
}
|
|
3293
|
+
this.deferredMessages.push(...restored);
|
|
3294
|
+
this.deferredWritesPersisted = true;
|
|
3295
|
+
// Rewrite the durable queue without the already-landed entries so a
|
|
3296
|
+
// second interrupted boot does not re-scan them.
|
|
3297
|
+
if (landed.size > 0) this.persistDeferredWrites();
|
|
3298
|
+
return restored.length;
|
|
3299
|
+
} catch (err) {
|
|
3300
|
+
console.error('[host-mode] failed to restore persisted deferred context writes:', err);
|
|
3301
|
+
return 0;
|
|
3302
|
+
}
|
|
3303
|
+
}
|
|
3304
|
+
|
|
3305
|
+
/**
|
|
3306
|
+
* Operator-driven maintenance: run (or join) one bounded compression pass —
|
|
3307
|
+
* the same runQueuedMaintenance the timer drives, so tool definitions are
|
|
3308
|
+
* refreshed and ticks are bounded/serialized identically — and return the
|
|
3309
|
+
* maintenance snapshot. Works while quiesced by design: `running` stays
|
|
3310
|
+
* true in quiesce mode precisely so this machinery stays hot.
|
|
3311
|
+
*/
|
|
3312
|
+
async maintenanceTick(): Promise<ContextMaintenanceSnapshot & { ran: boolean }> {
|
|
3313
|
+
if (!this.running) {
|
|
3314
|
+
// startQueuedMaintenance no-ops on a stopped/never-started framework —
|
|
3315
|
+
// a bare snapshot would masquerade as a completed pass, so say so.
|
|
3316
|
+
console.warn('[host-mode] maintenanceTick: framework is not running — no pass executed');
|
|
3317
|
+
return { ...this.getContextMaintenanceSnapshot(), ran: false };
|
|
3318
|
+
}
|
|
3319
|
+
this.startQueuedMaintenance();
|
|
3320
|
+
await this.maintenancePass;
|
|
3321
|
+
return { ...this.getContextMaintenanceSnapshot(), ran: true };
|
|
3322
|
+
}
|
|
3323
|
+
|
|
2202
3324
|
/** Counts-only context-maintenance diagnostics for authenticated debug UIs. */
|
|
2203
3325
|
getContextMaintenanceSnapshot(): ContextMaintenanceSnapshot {
|
|
2204
3326
|
const agents = [...this.agents.values()].map((agent) => {
|
|
@@ -2342,6 +3464,101 @@ export class AgentFramework {
|
|
|
2342
3464
|
);
|
|
2343
3465
|
}
|
|
2344
3466
|
|
|
3467
|
+
/**
|
|
3468
|
+
* Feasibility preview for a runtime-settings patch — or, with no patch, for
|
|
3469
|
+
* the agent's CURRENT settings (the resume-gate case: quiesce maintenance
|
|
3470
|
+
* may have moved the folded floor and the branch generation, so the verdict
|
|
3471
|
+
* must be computed fresh at resume time).
|
|
3472
|
+
*
|
|
3473
|
+
* Models the patch through the same immediate-vs-paced semantics as the
|
|
3474
|
+
* apply path (`Agent.planRuntimeSettings`): a non-immediate budget decrease
|
|
3475
|
+
* is previewed at the UNCHANGED live compile budget, with a best-effort
|
|
3476
|
+
* advisory verdict at the descent target. This is what the superseded
|
|
3477
|
+
* `feat/budget-preflight-guard` got wrong — it previewed the patch value as
|
|
3478
|
+
* the compile budget and rejected safe-by-construction paced descents.
|
|
3479
|
+
*
|
|
3480
|
+
* Never throws for preview-layer reasons: strategy without previewContext,
|
|
3481
|
+
* preview overlap, stale branch generation, and the hierarchical (non-
|
|
3482
|
+
* adaptive) path all come back as `{available: false, reason}`. Unknown
|
|
3483
|
+
* agent still throws, matching the sibling accessors.
|
|
3484
|
+
*/
|
|
3485
|
+
previewAgentRuntimeSettings(
|
|
3486
|
+
agentName: string,
|
|
3487
|
+
patch?: AgentRuntimeSettingsPatch,
|
|
3488
|
+
): RuntimeSettingsPreview {
|
|
3489
|
+
const agent = this.agents.get(agentName);
|
|
3490
|
+
if (!agent) throw new Error(`Unknown agent: ${agentName}`);
|
|
3491
|
+
const plan = agent.planRuntimeSettings(patch);
|
|
3492
|
+
const settings = agent.getRuntimeSettings();
|
|
3493
|
+
const base: RuntimeSettingsPreview = {
|
|
3494
|
+
available: false,
|
|
3495
|
+
path: plan.path,
|
|
3496
|
+
liveBudgetTokens: plan.liveBudgetTokens,
|
|
3497
|
+
transition: settings.transition,
|
|
3498
|
+
...(settings.transitionReason ? { transitionReason: settings.transitionReason } : {}),
|
|
3499
|
+
};
|
|
3500
|
+
|
|
3501
|
+
const runPreview = (budgetTokens: number): {
|
|
3502
|
+
result?: Record<string, unknown>;
|
|
3503
|
+
reason?: string;
|
|
3504
|
+
} => {
|
|
3505
|
+
try {
|
|
3506
|
+
const result = this.previewContextSettings(
|
|
3507
|
+
agentName,
|
|
3508
|
+
budgetTokens,
|
|
3509
|
+
Object.keys(plan.overrides).length > 0 ? plan.overrides : undefined,
|
|
3510
|
+
);
|
|
3511
|
+
if (result === null || typeof result !== 'object') {
|
|
3512
|
+
return { reason: 'no_preview_support' };
|
|
3513
|
+
}
|
|
3514
|
+
return { result: result as Record<string, unknown> };
|
|
3515
|
+
} catch (error) {
|
|
3516
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3517
|
+
// Structured mapping of the strategy's known throw paths. Message
|
|
3518
|
+
// matching is fragile by nature, so unknown messages surface verbatim
|
|
3519
|
+
// as the reason rather than being swallowed or rethrown.
|
|
3520
|
+
if (message.includes('requires reinitialization for the current branch generation')) {
|
|
3521
|
+
return { reason: 'branch_generation_changed' };
|
|
3522
|
+
}
|
|
3523
|
+
if (message.includes('already running; previews must not overlap')) {
|
|
3524
|
+
return { reason: 'preview_in_flight' };
|
|
3525
|
+
}
|
|
3526
|
+
if (message.includes('requires adaptiveResolution')) {
|
|
3527
|
+
return { reason: 'no_adaptive_resolution' };
|
|
3528
|
+
}
|
|
3529
|
+
return { reason: message };
|
|
3530
|
+
}
|
|
3531
|
+
};
|
|
3532
|
+
|
|
3533
|
+
const effective = runPreview(plan.effectiveBudgetTokens);
|
|
3534
|
+
if (!effective.result) {
|
|
3535
|
+
return { ...base, reason: effective.reason };
|
|
3536
|
+
}
|
|
3537
|
+
const preview: RuntimeSettingsPreview = {
|
|
3538
|
+
...base,
|
|
3539
|
+
available: true,
|
|
3540
|
+
effective: {
|
|
3541
|
+
...effective.result,
|
|
3542
|
+
budgetTokens: plan.effectiveBudgetTokens,
|
|
3543
|
+
fits: effective.result.fits === true,
|
|
3544
|
+
finalTokens: Number(effective.result.finalTokens ?? 0),
|
|
3545
|
+
},
|
|
3546
|
+
};
|
|
3547
|
+
if (plan.advisoryTargetTokens !== undefined) {
|
|
3548
|
+
// Best-effort: an advisory failure never flips `available` — the
|
|
3549
|
+
// effective verdict above is the one that gates anything.
|
|
3550
|
+
const advisory = runPreview(plan.advisoryTargetTokens);
|
|
3551
|
+
if (advisory.result) {
|
|
3552
|
+
preview.advisory = {
|
|
3553
|
+
...advisory.result,
|
|
3554
|
+
targetTokens: plan.advisoryTargetTokens,
|
|
3555
|
+
fits: advisory.result.fits === true,
|
|
3556
|
+
};
|
|
3557
|
+
}
|
|
3558
|
+
}
|
|
3559
|
+
return preview;
|
|
3560
|
+
}
|
|
3561
|
+
|
|
2345
3562
|
/**
|
|
2346
3563
|
* Check if process logging is enabled.
|
|
2347
3564
|
*/
|
|
@@ -2510,19 +3727,88 @@ export class AgentFramework {
|
|
|
2510
3727
|
this.store.setStateJson(FRAMEWORK_STATE_ID, state);
|
|
2511
3728
|
}
|
|
2512
3729
|
|
|
2513
|
-
private persistAgentRuntimeSettings(
|
|
2514
|
-
agentName: string,
|
|
2515
|
-
overrides: AgentRuntimeSettingsOverrides,
|
|
2516
|
-
): void {
|
|
2517
|
-
const data = this.store.getStateJson(FRAMEWORK_STATE_ID);
|
|
2518
|
-
const state = (data && typeof data === 'object' ? data : {}) as Record<string, unknown>;
|
|
2519
|
-
const all = {
|
|
2520
|
-
...((state.agentRuntimeSettings as Record<string, AgentRuntimeSettingsOverrides> | undefined) ?? {}),
|
|
2521
|
-
};
|
|
2522
|
-
if (Object.keys(overrides).length === 0) delete all[agentName];
|
|
2523
|
-
else all[agentName] = { ...overrides };
|
|
2524
|
-
state.agentRuntimeSettings = all;
|
|
2525
|
-
this.store.setStateJson(FRAMEWORK_STATE_ID, state);
|
|
3730
|
+
private persistAgentRuntimeSettings(
|
|
3731
|
+
agentName: string,
|
|
3732
|
+
overrides: AgentRuntimeSettingsOverrides,
|
|
3733
|
+
): void {
|
|
3734
|
+
const data = this.store.getStateJson(FRAMEWORK_STATE_ID);
|
|
3735
|
+
const state = (data && typeof data === 'object' ? data : {}) as Record<string, unknown>;
|
|
3736
|
+
const all = {
|
|
3737
|
+
...((state.agentRuntimeSettings as Record<string, AgentRuntimeSettingsOverrides> | undefined) ?? {}),
|
|
3738
|
+
};
|
|
3739
|
+
if (Object.keys(overrides).length === 0) delete all[agentName];
|
|
3740
|
+
else all[agentName] = { ...overrides };
|
|
3741
|
+
state.agentRuntimeSettings = all;
|
|
3742
|
+
this.store.setStateJson(FRAMEWORK_STATE_ID, state);
|
|
3743
|
+
}
|
|
3744
|
+
|
|
3745
|
+
/** Persist (or clear, with null) the host quiesce mode. First process-global
|
|
3746
|
+
* key in the framework/state slot: quiesce survives a restart BY DESIGN — a
|
|
3747
|
+
* crash mid-surgery must come back up NOT serving against a half-repaired
|
|
3748
|
+
* store. Resume is always reachable (public method, WS/HTTP, control-plane
|
|
3749
|
+
* host/command), and every quiesced boot logs a loud banner. */
|
|
3750
|
+
private persistHostMode(mode: { quiesced: true; reason?: string; since: number } | null): void {
|
|
3751
|
+
if (this.hostModePath) {
|
|
3752
|
+
// Branch-independent: the recovery file is the record. (`quiesced:
|
|
3753
|
+
// false` is written explicitly rather than deleting the file so that
|
|
3754
|
+
// its presence, not its content, decides precedence over the legacy
|
|
3755
|
+
// branch-local slot below.)
|
|
3756
|
+
this.writeRecoveryFile(
|
|
3757
|
+
this.hostModePath,
|
|
3758
|
+
mode === null ? { version: 1, quiesced: false, clearedAt: Date.now() } : { version: 1, ...mode },
|
|
3759
|
+
);
|
|
3760
|
+
if (mode === null) {
|
|
3761
|
+
// Also drop any legacy slot marker so a downgrade cannot resurrect it.
|
|
3762
|
+
try {
|
|
3763
|
+
const data = this.store.getStateJson(FRAMEWORK_STATE_ID);
|
|
3764
|
+
if (data && typeof data === 'object' && 'hostMode' in (data as Record<string, unknown>)) {
|
|
3765
|
+
const state = { ...(data as Record<string, unknown>) };
|
|
3766
|
+
delete state.hostMode;
|
|
3767
|
+
this.store.setStateJson(FRAMEWORK_STATE_ID, state);
|
|
3768
|
+
}
|
|
3769
|
+
} catch { /* best effort */ }
|
|
3770
|
+
}
|
|
3771
|
+
} else {
|
|
3772
|
+
this.warnRecoveryFallbackOnce();
|
|
3773
|
+
const data = this.store.getStateJson(FRAMEWORK_STATE_ID);
|
|
3774
|
+
const state = (data && typeof data === 'object' ? data : {}) as Record<string, unknown>;
|
|
3775
|
+
if (mode === null) delete state.hostMode;
|
|
3776
|
+
else state.hostMode = { ...mode };
|
|
3777
|
+
this.store.setStateJson(FRAMEWORK_STATE_ID, state);
|
|
3778
|
+
}
|
|
3779
|
+
// The whole feature rests on the flag surviving a crash — close the
|
|
3780
|
+
// window between the write and the next periodic sync.
|
|
3781
|
+
try {
|
|
3782
|
+
this.store.sync();
|
|
3783
|
+
} catch (err) {
|
|
3784
|
+
console.error('[host-mode] store sync after host-mode change failed:', err);
|
|
3785
|
+
}
|
|
3786
|
+
}
|
|
3787
|
+
|
|
3788
|
+
/** The recovery file wins whenever it exists; the branch-local slot is
|
|
3789
|
+
* read only for stores that predate it (or store-only configs). */
|
|
3790
|
+
private readHostMode(): { quiesced: boolean; reason?: string; since?: number } | null {
|
|
3791
|
+
try {
|
|
3792
|
+
let record: Record<string, unknown> | null = null;
|
|
3793
|
+
if (this.hostModePath && existsSync(this.hostModePath)) {
|
|
3794
|
+
const doc = this.readRecoveryFile(this.hostModePath);
|
|
3795
|
+
if (doc && typeof doc === 'object') record = doc as Record<string, unknown>;
|
|
3796
|
+
} else {
|
|
3797
|
+
const data = this.store.getStateJson(FRAMEWORK_STATE_ID);
|
|
3798
|
+
if (!data || typeof data !== 'object') return null;
|
|
3799
|
+
const mode = (data as Record<string, unknown>).hostMode;
|
|
3800
|
+
if (!mode || typeof mode !== 'object') return null;
|
|
3801
|
+
record = mode as Record<string, unknown>;
|
|
3802
|
+
}
|
|
3803
|
+
if (!record) return null;
|
|
3804
|
+
return {
|
|
3805
|
+
quiesced: record.quiesced === true,
|
|
3806
|
+
...(typeof record.reason === 'string' ? { reason: record.reason } : {}),
|
|
3807
|
+
...(typeof record.since === 'number' ? { since: record.since } : {}),
|
|
3808
|
+
};
|
|
3809
|
+
} catch {
|
|
3810
|
+
return null;
|
|
3811
|
+
}
|
|
2526
3812
|
}
|
|
2527
3813
|
|
|
2528
3814
|
/**
|
|
@@ -2650,18 +3936,40 @@ export class AgentFramework {
|
|
|
2650
3936
|
contextManager: ContextManager,
|
|
2651
3937
|
watchdogs?: { startupTimeoutMs?: number; idleTimeoutMs?: number; idlePollMs?: number },
|
|
2652
3938
|
): Promise<{ speech: string; toolCallsCount: number }> {
|
|
3939
|
+
// Refused at admission while quiesced rather than requeued: the caller
|
|
3940
|
+
// awaits the settle promise under an idle-stall watchdog, so a gate-parked
|
|
3941
|
+
// ephemeral would ride out its timeout and reject with a misleading
|
|
3942
|
+
// "stalled" error hours later. A clear refusal now beats that. Checked
|
|
3943
|
+
// before the generation ticket is consumed so the same (agent, cm) pair
|
|
3944
|
+
// can be retried after resume().
|
|
3945
|
+
if (this.quiesced) {
|
|
3946
|
+
throw new Error(
|
|
3947
|
+
`framework is quiesced${this.quiesceReason ? ` (${this.quiesceReason})` : ''}: ` +
|
|
3948
|
+
`refusing new ephemeral run for ${agent.name} — resume() first`,
|
|
3949
|
+
);
|
|
3950
|
+
}
|
|
2653
3951
|
// Only a fresh object returned by createEphemeralAgent may enter this path.
|
|
2654
3952
|
// Never overwrite a resident/conversation owner or a concurrent run: their
|
|
2655
3953
|
// cleanup is name-keyed and could cancel/deregister the legitimate owner.
|
|
2656
3954
|
if (this.ephemeralCandidates.get(agent) !== contextManager) {
|
|
2657
3955
|
throw new Error(`Ephemeral agent "${agent.name}" has no fresh generation ticket from this framework`);
|
|
2658
3956
|
}
|
|
3957
|
+
// A live surgery holds the whole store: an agent admitted now would
|
|
3958
|
+
// compile against a history that is being replaced under it. Refuse
|
|
3959
|
+
// BEFORE consuming the ticket so the caller can retry once it completes.
|
|
3960
|
+
if (this.surgeryHold) {
|
|
3961
|
+
throw new Error(
|
|
3962
|
+
`Ephemeral agent "${agent.name}" refused: the store is under live ${this.surgeryHold.verb} ` +
|
|
3963
|
+
`for ${this.surgeryHold.agentName} — retry when it completes`,
|
|
3964
|
+
);
|
|
3965
|
+
}
|
|
2659
3966
|
// Consume before any await/registration: the exact (Agent, ContextManager)
|
|
2660
3967
|
// generation is one-shot even if startup later fails.
|
|
2661
3968
|
this.ephemeralCandidates.delete(agent);
|
|
2662
3969
|
if (this.agents.has(agent.name) || this.ephemeralRuns.has(agent.name)) {
|
|
2663
3970
|
throw new Error(`Ephemeral agent "${agent.name}" is already registered or running`);
|
|
2664
3971
|
}
|
|
3972
|
+
// Register temporarily so the event loop can drive it
|
|
2665
3973
|
this.agents.set(agent.name, agent);
|
|
2666
3974
|
const run: EphemeralRun = {
|
|
2667
3975
|
settle: this.createDeferred<AgentSettleResult>(),
|
|
@@ -3406,6 +4714,8 @@ export class AgentFramework {
|
|
|
3406
4714
|
/** Agent state at queue time — 'idle' means the turn starts on the next
|
|
3407
4715
|
* scheduler pass; anything else means it runs after the current turn. */
|
|
3408
4716
|
agentStatus?: string;
|
|
4717
|
+
/** True when the host is quiesced: the wake is PARKED, not running, until resume(). */
|
|
4718
|
+
quiesced?: boolean;
|
|
3409
4719
|
} {
|
|
3410
4720
|
const name = agentName ?? [...this.agents.keys()][0];
|
|
3411
4721
|
const agent = name ? this.agents.get(name) : undefined;
|
|
@@ -3421,8 +4731,17 @@ export class AgentFramework {
|
|
|
3421
4731
|
source: 'admin',
|
|
3422
4732
|
timestamp: Date.now(),
|
|
3423
4733
|
});
|
|
3424
|
-
console.error(
|
|
3425
|
-
|
|
4734
|
+
console.error(
|
|
4735
|
+
`[nudge] agent=${name} queued by=${requestedBy ?? 'unknown'} status=${agentStatus}` +
|
|
4736
|
+
(this.quiesced ? ' (host quiesced — parked until resume)' : ''),
|
|
4737
|
+
);
|
|
4738
|
+
this.recordOperatorAction({
|
|
4739
|
+
kind: 'nudge',
|
|
4740
|
+
agent: name,
|
|
4741
|
+
requester: { via: 'nudge', ...(requestedBy ? { name: requestedBy } : {}) },
|
|
4742
|
+
result: { agentStatus, ...(this.quiesced ? { quiesced: true } : {}) },
|
|
4743
|
+
});
|
|
4744
|
+
return { ok: true, agentName: name, agentStatus, ...(this.quiesced ? { quiesced: true } : {}) };
|
|
3426
4745
|
}
|
|
3427
4746
|
|
|
3428
4747
|
/**
|
|
@@ -3436,6 +4755,20 @@ export class AgentFramework {
|
|
|
3436
4755
|
*
|
|
3437
4756
|
* nudge — run inference on the current context with NO new events
|
|
3438
4757
|
* (see `nudgeAgent`).
|
|
4758
|
+
*
|
|
4759
|
+
* quiesce — enter host maintenance mode (issue #122): park wakes, pause
|
|
4760
|
+
* MCPL data planes, drain in-flight turns (optionally `abandon` on
|
|
4761
|
+
* timeout). Host-scoped, no agentName.
|
|
4762
|
+
*
|
|
4763
|
+
* resume — leave maintenance mode behind a fresh feasibility preview of
|
|
4764
|
+
* every agent's current settings (`force` overrides a failing verdict).
|
|
4765
|
+
*
|
|
4766
|
+
* maintain — run (or join) one bounded compression/maintenance pass and
|
|
4767
|
+
* return the snapshot; the operator lever for draining quarantine or
|
|
4768
|
+
* advancing merges during a quiesce window.
|
|
4769
|
+
*
|
|
4770
|
+
* host-status — report the quiesce state (drained / parked wakes /
|
|
4771
|
+
* running background scripts).
|
|
3439
4772
|
*/
|
|
3440
4773
|
private async handleHostCommand(
|
|
3441
4774
|
serverId: string,
|
|
@@ -3443,8 +4776,14 @@ export class AgentFramework {
|
|
|
3443
4776
|
): Promise<{
|
|
3444
4777
|
ok: boolean;
|
|
3445
4778
|
error?: string;
|
|
4779
|
+
/** Refusal code from live surgery (e.g. 'agent-busy'), when applicable. */
|
|
4780
|
+
code?: string;
|
|
3446
4781
|
undone?: number;
|
|
3447
4782
|
requested?: number;
|
|
4783
|
+
/** For quiesce/resume/maintain/host-status: the host mode snapshot. */
|
|
4784
|
+
hostMode?: HostModeStatus;
|
|
4785
|
+
/** For `maintain`: the context-maintenance snapshot after the pass. */
|
|
4786
|
+
maintenance?: ContextMaintenanceSnapshot;
|
|
3448
4787
|
messagesRemoved?: number;
|
|
3449
4788
|
/** Discord addresses removed by message-granular undo. The durable outbox
|
|
3450
4789
|
* owns eventual delivery; this is also returned for immediate surfaces. */
|
|
@@ -3458,6 +4797,9 @@ export class AgentFramework {
|
|
|
3458
4797
|
* outcome report is posted to the channel asynchronously. */
|
|
3459
4798
|
started?: boolean;
|
|
3460
4799
|
cap?: number;
|
|
4800
|
+
/** For `nudge`/`unstick`: the host is quiesced, so the request is parked
|
|
4801
|
+
* until resume rather than running. */
|
|
4802
|
+
quiesced?: boolean;
|
|
3461
4803
|
/** For `nudge`: agent state at queue time ('idle' = runs immediately,
|
|
3462
4804
|
* else it runs once the current turn settles). */
|
|
3463
4805
|
agentStatus?: string;
|
|
@@ -3466,11 +4808,48 @@ export class AgentFramework {
|
|
|
3466
4808
|
params.command !== 'undo' &&
|
|
3467
4809
|
params.command !== 'hide' &&
|
|
3468
4810
|
params.command !== 'unstick' &&
|
|
3469
|
-
params.command !== 'nudge'
|
|
4811
|
+
params.command !== 'nudge' &&
|
|
4812
|
+
params.command !== 'quiesce' &&
|
|
4813
|
+
params.command !== 'resume' &&
|
|
4814
|
+
params.command !== 'maintain' &&
|
|
4815
|
+
params.command !== 'host-status'
|
|
3470
4816
|
) {
|
|
3471
4817
|
return { ok: false, error: `Unknown host command: ${String(params.command)}` };
|
|
3472
4818
|
}
|
|
3473
4819
|
|
|
4820
|
+
// Host-scoped verbs (issue #122) — no agent resolution.
|
|
4821
|
+
const requester = params.requesterName ?? params.requesterId ?? `mcpl:${serverId}`;
|
|
4822
|
+
if (params.command === 'quiesce') {
|
|
4823
|
+
// quiesce() clamps timeoutMs to [1s, 10m] itself — one clamp for every ingress.
|
|
4824
|
+
console.error(`[host-command] quiesce by=${requester} (server=${serverId})`);
|
|
4825
|
+
const hostMode = await this.quiesce({
|
|
4826
|
+
reason: params.reason ?? `host/command by ${requester}`,
|
|
4827
|
+
...(params.timeoutMs !== undefined ? { timeoutMs: params.timeoutMs } : {}),
|
|
4828
|
+
...(params.abandon ? { abandon: true } : {}),
|
|
4829
|
+
});
|
|
4830
|
+
return { ok: true, hostMode };
|
|
4831
|
+
}
|
|
4832
|
+
if (params.command === 'resume') {
|
|
4833
|
+
console.error(`[host-command] resume by=${requester} (server=${serverId})`);
|
|
4834
|
+
try {
|
|
4835
|
+
const hostMode = await this.resume(params.force ? { force: true } : undefined);
|
|
4836
|
+
return { ok: true, hostMode };
|
|
4837
|
+
} catch (error) {
|
|
4838
|
+
if (error instanceof ResumeBlockedError) {
|
|
4839
|
+
return { ok: false, error: error.message, hostMode: this.getHostModeStatus() };
|
|
4840
|
+
}
|
|
4841
|
+
throw error;
|
|
4842
|
+
}
|
|
4843
|
+
}
|
|
4844
|
+
if (params.command === 'maintain') {
|
|
4845
|
+
console.error(`[host-command] maintain by=${requester} (server=${serverId})`);
|
|
4846
|
+
const maintenance = await this.maintenanceTick();
|
|
4847
|
+
return { ok: true, maintenance, hostMode: this.getHostModeStatus() };
|
|
4848
|
+
}
|
|
4849
|
+
if (params.command === 'host-status') {
|
|
4850
|
+
return { ok: true, hostMode: this.getHostModeStatus() };
|
|
4851
|
+
}
|
|
4852
|
+
|
|
3474
4853
|
const agentName = params.agentName ?? [...this.agents.keys()][0];
|
|
3475
4854
|
if (!agentName || !this.agents.has(agentName)) {
|
|
3476
4855
|
return { ok: false, error: `Unknown agent: ${String(agentName)}` };
|
|
@@ -3483,7 +4862,7 @@ export class AgentFramework {
|
|
|
3483
4862
|
agentName,
|
|
3484
4863
|
params.requesterName ?? params.requesterId ?? `mcpl:${serverId}`,
|
|
3485
4864
|
);
|
|
3486
|
-
return { ok: r.ok, error: r.error, agentStatus: r.agentStatus };
|
|
4865
|
+
return { ok: r.ok, error: r.error, agentStatus: r.agentStatus, ...(r.quiesced ? { quiesced: true } : {}) };
|
|
3487
4866
|
}
|
|
3488
4867
|
|
|
3489
4868
|
// unstick: force the refusal-rewind loop on demand (even if the agent's
|
|
@@ -3515,7 +4894,14 @@ export class AgentFramework {
|
|
|
3515
4894
|
`[unstick] agent=${agentName} started cap=${cap} ` +
|
|
3516
4895
|
`by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`,
|
|
3517
4896
|
);
|
|
3518
|
-
|
|
4897
|
+
this.recordOperatorAction({
|
|
4898
|
+
kind: 'unstick',
|
|
4899
|
+
agent: agentName,
|
|
4900
|
+
requester: hostCommandRequester(serverId, params),
|
|
4901
|
+
params: { cap },
|
|
4902
|
+
result: { started: true, ...(this.quiesced ? { quiesced: true } : {}) },
|
|
4903
|
+
});
|
|
4904
|
+
return { ok: true, started: true, cap, ...(this.quiesced ? { quiesced: true } : {}) };
|
|
3519
4905
|
}
|
|
3520
4906
|
|
|
3521
4907
|
// hide: redact a single message (or an inclusive range) from the active
|
|
@@ -3565,11 +4951,25 @@ export class AgentFramework {
|
|
|
3565
4951
|
}
|
|
3566
4952
|
const [lo, hi] = fromIdx <= toIdx ? [fromIdx, toIdx] : [toIdx, fromIdx];
|
|
3567
4953
|
const refs = refsIn(lo, hi);
|
|
4954
|
+
const rangeIds = all.slice(lo, hi + 1).map((m) => String(m.id));
|
|
3568
4955
|
cm.removeMessages(all[lo].id, all[hi].id);
|
|
3569
4956
|
console.error(
|
|
3570
4957
|
`[host-command] hide agent=${agentName} range removed=${hi - lo + 1} ` +
|
|
3571
4958
|
`(${params.fromMessageId}..${params.toMessageId}) by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`,
|
|
3572
4959
|
);
|
|
4960
|
+
const loggedRange = capIds(rangeIds);
|
|
4961
|
+
this.recordOperatorAction({
|
|
4962
|
+
kind: 'hide',
|
|
4963
|
+
agent: agentName,
|
|
4964
|
+
requester: hostCommandRequester(serverId, params),
|
|
4965
|
+
params: { fromMessageId: params.fromMessageId, toMessageId: params.toMessageId },
|
|
4966
|
+
result: {
|
|
4967
|
+
branch: this.store.currentBranch().name,
|
|
4968
|
+
hidden: hi - lo + 1,
|
|
4969
|
+
removedIds: loggedRange.ids,
|
|
4970
|
+
...(loggedRange.truncated ? { removedIdsTruncated: true } : {}),
|
|
4971
|
+
},
|
|
4972
|
+
});
|
|
3573
4973
|
return {
|
|
3574
4974
|
ok: true,
|
|
3575
4975
|
hidden: hi - lo + 1,
|
|
@@ -3578,11 +4978,19 @@ export class AgentFramework {
|
|
|
3578
4978
|
};
|
|
3579
4979
|
}
|
|
3580
4980
|
const refs = refsIn(fromIdx, fromIdx);
|
|
4981
|
+
const hiddenId = String(all[fromIdx].id);
|
|
3581
4982
|
cm.removeMessage(all[fromIdx].id);
|
|
3582
4983
|
console.error(
|
|
3583
4984
|
`[host-command] hide agent=${agentName} removed=1 (${params.fromMessageId}) ` +
|
|
3584
4985
|
`by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`,
|
|
3585
4986
|
);
|
|
4987
|
+
this.recordOperatorAction({
|
|
4988
|
+
kind: 'hide',
|
|
4989
|
+
agent: agentName,
|
|
4990
|
+
requester: hostCommandRequester(serverId, params),
|
|
4991
|
+
params: { fromMessageId: params.fromMessageId },
|
|
4992
|
+
result: { branch: this.store.currentBranch().name, hidden: 1, removedIds: [hiddenId] },
|
|
4993
|
+
});
|
|
3586
4994
|
return {
|
|
3587
4995
|
ok: true,
|
|
3588
4996
|
hidden: 1,
|
|
@@ -3605,62 +5013,40 @@ export class AgentFramework {
|
|
|
3605
5013
|
}
|
|
3606
5014
|
const n = Math.max(1, Math.min(50, Math.floor(params.messages)));
|
|
3607
5015
|
const cm = agent.getContextManager();
|
|
3608
|
-
const
|
|
3609
|
-
if (n >=
|
|
5016
|
+
const total = cm.getMessageCount();
|
|
5017
|
+
if (n >= total) {
|
|
3610
5018
|
return {
|
|
3611
5019
|
ok: false,
|
|
3612
|
-
error: `Cannot remove ${n} message(s) — history has ${
|
|
5020
|
+
error: `Cannot remove ${n} message(s) — history has ${total}; at least one must remain.`,
|
|
3613
5021
|
};
|
|
3614
5022
|
}
|
|
3615
|
-
const target =
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
|
|
3622
|
-
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
|
|
3626
|
-
|
|
3627
|
-
|
|
3628
|
-
|
|
3629
|
-
|
|
3630
|
-
|
|
3631
|
-
|
|
3632
|
-
|
|
3633
|
-
|
|
3634
|
-
if (markerBatch) this.discordAwarenessOutbox!.activate(markerBatch.id);
|
|
3635
|
-
await this.syncDiscordAwarenessMarkers();
|
|
3636
|
-
|
|
3637
|
-
// Materialize config files from the new branch (fire-and-forget; gate
|
|
3638
|
-
// picks up via mtime) — mirrors undoLastTurn.
|
|
3639
|
-
const wsUndo = this.moduleRegistry.getModule('workspace');
|
|
3640
|
-
if (wsUndo && 'materializeMount' in wsUndo) {
|
|
3641
|
-
(wsUndo as { materializeMount: (m: string) => Promise<void> })
|
|
3642
|
-
.materializeMount('_config')
|
|
3643
|
-
.catch(() => {});
|
|
5023
|
+
const target = cm.getMessageWindow(total - 1 - n, 1, { resolveBlobs: false }).messages[0];
|
|
5024
|
+
try {
|
|
5025
|
+
const r = await this.rollbackToMessage(agentName, {
|
|
5026
|
+
messageId: String(target.id),
|
|
5027
|
+
branchName: `undo-msgs/${agentName}/${Date.now()}`,
|
|
5028
|
+
requester: hostCommandRequester(serverId, params),
|
|
5029
|
+
});
|
|
5030
|
+
return {
|
|
5031
|
+
ok: true,
|
|
5032
|
+
messagesRemoved: r.messagesRemoved,
|
|
5033
|
+
removedRefs: r.removedRefs,
|
|
5034
|
+
lastVisible: r.lastVisible,
|
|
5035
|
+
};
|
|
5036
|
+
} catch (error) {
|
|
5037
|
+
return {
|
|
5038
|
+
ok: false,
|
|
5039
|
+
error: error instanceof Error ? error.message : String(error),
|
|
5040
|
+
...(error instanceof OperatorActionError ? { code: error.code } : {}),
|
|
5041
|
+
};
|
|
3644
5042
|
}
|
|
3645
|
-
|
|
3646
|
-
console.error(
|
|
3647
|
-
`[host-command] undo-messages agent=${agentName} removed=${n} branch=${branchName}` +
|
|
3648
|
-
` by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`,
|
|
3649
|
-
);
|
|
3650
|
-
|
|
3651
|
-
return {
|
|
3652
|
-
ok: true,
|
|
3653
|
-
messagesRemoved: n,
|
|
3654
|
-
removedRefs,
|
|
3655
|
-
lastVisible: await this.lastVisiblePreview(agentName),
|
|
3656
|
-
};
|
|
3657
5043
|
}
|
|
3658
5044
|
|
|
3659
5045
|
const requested = Math.max(1, Math.min(20, Math.floor(params.turns ?? 1)));
|
|
3660
5046
|
let undone = 0;
|
|
3661
5047
|
try {
|
|
3662
5048
|
for (let i = 0; i < requested; i++) {
|
|
3663
|
-
const r = this.undoLastTurn(agentName);
|
|
5049
|
+
const r = this.undoLastTurn(agentName, hostCommandRequester(serverId, params));
|
|
3664
5050
|
if (!r.undone) break;
|
|
3665
5051
|
undone++;
|
|
3666
5052
|
}
|
|
@@ -3998,7 +5384,7 @@ export class AgentFramework {
|
|
|
3998
5384
|
* The undone branch is saved so `redo()` can restore it.
|
|
3999
5385
|
* Returns the checkpoint that was undone, or null if nothing to undo.
|
|
4000
5386
|
*/
|
|
4001
|
-
undoLastTurn(agentName: string): {
|
|
5387
|
+
undoLastTurn(agentName: string, requester?: OperatorRequester): {
|
|
4002
5388
|
undone: boolean;
|
|
4003
5389
|
turnIndex?: number;
|
|
4004
5390
|
fromBranch?: string;
|
|
@@ -4047,6 +5433,13 @@ export class AgentFramework {
|
|
|
4047
5433
|
fromBranch: currentBranch.name,
|
|
4048
5434
|
toBranch: undoBranchName,
|
|
4049
5435
|
});
|
|
5436
|
+
this.recordOperatorAction({
|
|
5437
|
+
kind: 'undo-turn',
|
|
5438
|
+
agent: agentName,
|
|
5439
|
+
...(requester ? { requester } : {}),
|
|
5440
|
+
params: { turnIndex: checkpoint.turnIndex },
|
|
5441
|
+
result: { sourceBranch: currentBranch.name, targetBranch: undoBranchName },
|
|
5442
|
+
});
|
|
4050
5443
|
|
|
4051
5444
|
return {
|
|
4052
5445
|
undone: true,
|
|
@@ -4062,7 +5455,7 @@ export class AgentFramework {
|
|
|
4062
5455
|
* Switches back to the branch that was active before the last undo.
|
|
4063
5456
|
* Returns false if there's nothing to redo.
|
|
4064
5457
|
*/
|
|
4065
|
-
redo(agentName: string): {
|
|
5458
|
+
redo(agentName: string, requester?: OperatorRequester): {
|
|
4066
5459
|
redone: boolean;
|
|
4067
5460
|
fromBranch?: string;
|
|
4068
5461
|
toBranch?: string;
|
|
@@ -4102,6 +5495,12 @@ export class AgentFramework {
|
|
|
4102
5495
|
fromBranch: currentBranch.name,
|
|
4103
5496
|
toBranch: branchName,
|
|
4104
5497
|
});
|
|
5498
|
+
this.recordOperatorAction({
|
|
5499
|
+
kind: 'redo-turn',
|
|
5500
|
+
agent: agentName,
|
|
5501
|
+
...(requester ? { requester } : {}),
|
|
5502
|
+
result: { sourceBranch: currentBranch.name, targetBranch: branchName },
|
|
5503
|
+
});
|
|
4105
5504
|
|
|
4106
5505
|
return {
|
|
4107
5506
|
redone: true,
|
|
@@ -4110,6 +5509,472 @@ export class AgentFramework {
|
|
|
4110
5509
|
};
|
|
4111
5510
|
}
|
|
4112
5511
|
|
|
5512
|
+
// ---------------------------------------------------------------------------
|
|
5513
|
+
// Live operator surgery — rollback / suppress on the OPEN store, without a
|
|
5514
|
+
// restart — plus the durable action log every operator mutation writes to.
|
|
5515
|
+
//
|
|
5516
|
+
// Both follow the same idiom the offline surgeries use (recovery/
|
|
5517
|
+
// offline-branch.ts): fork first, mutate the fork, and make the fork the
|
|
5518
|
+
// active branch. The parent branch keeps everything, so restore is a
|
|
5519
|
+
// checkout. Mutations are refused (never queued) while the agent is not
|
|
5520
|
+
// idle — quiesce the host first if it is busy.
|
|
5521
|
+
// ---------------------------------------------------------------------------
|
|
5522
|
+
|
|
5523
|
+
/**
|
|
5524
|
+
* Append a record to the operator action log and broadcast it as an
|
|
5525
|
+
* `operator:action` trace. Hosts use this directly for actions the
|
|
5526
|
+
* framework does not own (e.g. a WebUI toggling quiesce). Never throws.
|
|
5527
|
+
*/
|
|
5528
|
+
recordOperatorAction(input: OperatorLogInput): OperatorLogEntry {
|
|
5529
|
+
// Observability never fails the action: OperatorLog.append never throws,
|
|
5530
|
+
// and a misbehaving trace listener is contained here.
|
|
5531
|
+
const entry = this.operatorLog.append(input);
|
|
5532
|
+
try {
|
|
5533
|
+
this.emitTrace({
|
|
5534
|
+
type: 'operator:action',
|
|
5535
|
+
kind: entry.kind,
|
|
5536
|
+
...(entry.agent !== undefined ? { agentName: entry.agent } : {}),
|
|
5537
|
+
...(entry.requester !== undefined ? { requester: entry.requester } : {}),
|
|
5538
|
+
...(entry.note !== undefined ? { note: entry.note } : {}),
|
|
5539
|
+
...(entry.params !== undefined ? { params: entry.params } : {}),
|
|
5540
|
+
...(entry.result !== undefined ? { result: entry.result } : {}),
|
|
5541
|
+
...(entry.error !== undefined ? { error: entry.error } : {}),
|
|
5542
|
+
});
|
|
5543
|
+
} catch {
|
|
5544
|
+
// swallowed by design
|
|
5545
|
+
}
|
|
5546
|
+
return entry;
|
|
5547
|
+
}
|
|
5548
|
+
|
|
5549
|
+
/** Newest `limit` operator-log entries, oldest first. */
|
|
5550
|
+
getOperatorLog(opts?: { limit?: number }): OperatorLogEntry[] {
|
|
5551
|
+
return this.operatorLog.readTail(opts?.limit ?? 100);
|
|
5552
|
+
}
|
|
5553
|
+
|
|
5554
|
+
/** Where the operator log is written, or undefined when disabled. */
|
|
5555
|
+
getOperatorLogPath(): string | undefined {
|
|
5556
|
+
return this.operatorLog.path;
|
|
5557
|
+
}
|
|
5558
|
+
|
|
5559
|
+
/**
|
|
5560
|
+
* The surgery gate — STORE-wide, not per agent. A branch switch moves the
|
|
5561
|
+
* active chronicle branch for every context manager sharing this store,
|
|
5562
|
+
* so an in-flight turn on *any* agent would have its next writes land on
|
|
5563
|
+
* a history its request was never compiled against. Every agent must be
|
|
5564
|
+
* idle with no turn alive (`state.status === 'idle'` alone is not enough:
|
|
5565
|
+
* status reads idle from dequeue until the stream registers and again
|
|
5566
|
+
* during teardown — the scheduler's own busy test is status + turn-alive,
|
|
5567
|
+
* the predicate `puppetToolCall` uses).
|
|
5568
|
+
*
|
|
5569
|
+
* Passing the gate RESERVES the store for the caller: a turn token is
|
|
5570
|
+
* held for every agent until the returned release runs, so no wake — via
|
|
5571
|
+
* the scheduler or a parked provider admission, both of which re-test
|
|
5572
|
+
* turn-alive — can start a turn while the switch is awaited. Release also
|
|
5573
|
+
* flushes messages that deferred behind the reservation, as the puppet
|
|
5574
|
+
* does. Always release in `finally`: a token nobody clears wedges the fleet.
|
|
5575
|
+
*/
|
|
5576
|
+
private reserveStoreForSurgery(verb: string, agentName: string): () => void {
|
|
5577
|
+
if (this.surgeryHold) {
|
|
5578
|
+
throw new OperatorActionError(
|
|
5579
|
+
'agent-busy',
|
|
5580
|
+
`Cannot ${verb}: a live ${this.surgeryHold.verb} for ${this.surgeryHold.agentName} is already in progress`,
|
|
5581
|
+
);
|
|
5582
|
+
}
|
|
5583
|
+
const busy: string[] = [];
|
|
5584
|
+
for (const [name, a] of this.agents) {
|
|
5585
|
+
if (a.state.status !== 'idle') busy.push(`${name} is ${a.state.status}`);
|
|
5586
|
+
else if (this.activeTurnTokens.has(name)) busy.push(`${name} is idle+turn-alive`);
|
|
5587
|
+
}
|
|
5588
|
+
for (const name of this.activeTurnTokens.keys()) {
|
|
5589
|
+
if (!this.agents.has(name)) busy.push(`${name} is turn-alive`);
|
|
5590
|
+
}
|
|
5591
|
+
if (busy.length > 0) {
|
|
5592
|
+
throw new OperatorActionError(
|
|
5593
|
+
'agent-busy',
|
|
5594
|
+
`Cannot ${verb} while ${busy.join(', ')} — every agent sharing the store must be idle (quiesce the host first)`,
|
|
5595
|
+
);
|
|
5596
|
+
}
|
|
5597
|
+
const reserved = new Map<string, number>();
|
|
5598
|
+
for (const name of this.agents.keys()) {
|
|
5599
|
+
const token = this.nextTurnToken++;
|
|
5600
|
+
this.activeTurnTokens.set(name, token);
|
|
5601
|
+
reserved.set(name, token);
|
|
5602
|
+
}
|
|
5603
|
+
// The token snapshot only covers agents that exist now; the hold covers
|
|
5604
|
+
// arrivals during the awaited switch (see surgeryHold).
|
|
5605
|
+
const hold = { verb, agentName, since: Date.now() };
|
|
5606
|
+
this.surgeryHold = hold;
|
|
5607
|
+
let released = false;
|
|
5608
|
+
return () => {
|
|
5609
|
+
if (released) return;
|
|
5610
|
+
released = true;
|
|
5611
|
+
if (this.surgeryHold === hold) this.surgeryHold = null;
|
|
5612
|
+
for (const [name, token] of reserved) {
|
|
5613
|
+
if (this.activeTurnTokens.get(name) === token) this.activeTurnTokens.delete(name);
|
|
5614
|
+
}
|
|
5615
|
+
// Writers that deferred behind the reservation land now, on the branch
|
|
5616
|
+
// the operator chose (or the restored source) — the same end-of-turn
|
|
5617
|
+
// flush driveStream/puppet perform, under the same guard.
|
|
5618
|
+
if (this.deferredMessages.length > 0 && this.pendingAssistantBlocks.size === 0) {
|
|
5619
|
+
for (const name of reserved.keys()) {
|
|
5620
|
+
if (this.activeTurnTokens.has(name)) continue;
|
|
5621
|
+
for (const msg of this.drainDeferredFor(name)) {
|
|
5622
|
+
try {
|
|
5623
|
+
this.addMessage(msg.participant, msg.content, msg.metadata,
|
|
5624
|
+
msg.forAgent ? { forAgent: msg.forAgent } : undefined);
|
|
5625
|
+
} catch (error) {
|
|
5626
|
+
console.error(`[operator] post-surgery deferred flush failed for ${name}: ${error instanceof Error ? error.message : String(error)}`);
|
|
5627
|
+
}
|
|
5628
|
+
}
|
|
5629
|
+
}
|
|
5630
|
+
}
|
|
5631
|
+
};
|
|
5632
|
+
}
|
|
5633
|
+
|
|
5634
|
+
/**
|
|
5635
|
+
* After a failed switch or redaction: put the store back on `sourceBranch`
|
|
5636
|
+
* if it moved, and return a sentence saying what actually happened. Never
|
|
5637
|
+
* throws — the caller reports the original failure with this appended.
|
|
5638
|
+
*/
|
|
5639
|
+
private async restoreSourceBranch(
|
|
5640
|
+
cm: ContextManager,
|
|
5641
|
+
sourceBranch: string,
|
|
5642
|
+
failedBranch: string,
|
|
5643
|
+
agentName: string,
|
|
5644
|
+
): Promise<string> {
|
|
5645
|
+
const current = this.store.currentBranch().name;
|
|
5646
|
+
if (current !== failedBranch) return `active branch is ${current}`;
|
|
5647
|
+
try {
|
|
5648
|
+
await cm.switchBranch(sourceBranch);
|
|
5649
|
+
return `active branch restored to ${sourceBranch}`;
|
|
5650
|
+
} catch (restoreError) {
|
|
5651
|
+
const detail = restoreError instanceof Error ? restoreError.message : String(restoreError);
|
|
5652
|
+
const status = `RESTORE TO ${sourceBranch} ALSO FAILED (${detail}); agent is still on ${failedBranch}`;
|
|
5653
|
+
console.error(`[operator] agent=${agentName} ${status}`);
|
|
5654
|
+
return status;
|
|
5655
|
+
}
|
|
5656
|
+
}
|
|
5657
|
+
|
|
5658
|
+
/**
|
|
5659
|
+
* Roll the active branch back so `messageId` becomes its tail: fork the
|
|
5660
|
+
* chronicle at that message (origin-sequence time-travel branch) and switch
|
|
5661
|
+
* to the fork. Everything after the message stays on the source branch.
|
|
5662
|
+
* Discord messages that left the live context get awareness markers via the
|
|
5663
|
+
* durable outbox, exactly as message-granular `undo` does.
|
|
5664
|
+
*
|
|
5665
|
+
* Throws `OperatorActionError` (`agent-busy`, `unknown-message`, …) — the
|
|
5666
|
+
* agent must be idle; nothing is queued.
|
|
5667
|
+
*/
|
|
5668
|
+
async rollbackToMessage(
|
|
5669
|
+
agentName: string,
|
|
5670
|
+
opts: {
|
|
5671
|
+
messageId: string;
|
|
5672
|
+
requester?: OperatorRequester;
|
|
5673
|
+
note?: string;
|
|
5674
|
+
/** Branch name for the fork (default `rollback/<agent>/<ts>`). */
|
|
5675
|
+
branchName?: string;
|
|
5676
|
+
},
|
|
5677
|
+
): Promise<{
|
|
5678
|
+
agentName: string;
|
|
5679
|
+
sourceBranch: string;
|
|
5680
|
+
targetBranch: string;
|
|
5681
|
+
messagesRemoved: number;
|
|
5682
|
+
/** The message that actually became the tail. Differs from the requested
|
|
5683
|
+
* id only when that id was a shard of a body group: the fork lands on
|
|
5684
|
+
* the group's last shard so the sharded message stays whole. */
|
|
5685
|
+
tailMessageId: string;
|
|
5686
|
+
removedRefs: Array<{ serverId: string; channelId: string; messageId: string }>;
|
|
5687
|
+
lastVisible: { participant?: string; role?: string; preview?: string } | null;
|
|
5688
|
+
}> {
|
|
5689
|
+
const logBase: OperatorLogInput = {
|
|
5690
|
+
kind: 'rollback',
|
|
5691
|
+
agent: agentName,
|
|
5692
|
+
...(opts.requester ? { requester: opts.requester } : {}),
|
|
5693
|
+
...(opts.note ? { note: opts.note } : {}),
|
|
5694
|
+
params: { messageId: opts.messageId },
|
|
5695
|
+
};
|
|
5696
|
+
try {
|
|
5697
|
+
const agent = this.agents.get(agentName);
|
|
5698
|
+
if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${agentName}`);
|
|
5699
|
+
const cm = agent.getContextManager();
|
|
5700
|
+
const total = cm.getMessageCount();
|
|
5701
|
+
const requestedIndex = locateMessageIndex(cm, opts.messageId);
|
|
5702
|
+
if (requestedIndex < 0) {
|
|
5703
|
+
throw new OperatorActionError('unknown-message', `Message ${opts.messageId} is not on the active branch`);
|
|
5704
|
+
}
|
|
5705
|
+
// Never bisect a body group: chronicle refuses that for removals
|
|
5706
|
+
// (byte-faithful reassembly needs the whole run) and branchAt has no
|
|
5707
|
+
// such guard of its own. Snap to the run's last shard.
|
|
5708
|
+
const run = bodyGroupRun(cm, requestedIndex);
|
|
5709
|
+
const targetIndex = run.to;
|
|
5710
|
+
const tailMessageId = run.toId;
|
|
5711
|
+
const messagesRemoved = total - targetIndex - 1;
|
|
5712
|
+
if (messagesRemoved <= 0) {
|
|
5713
|
+
throw new OperatorActionError('invalid', `Message ${opts.messageId} is already the tail of the active branch`);
|
|
5714
|
+
}
|
|
5715
|
+
const discarded = cm.getMessageWindow(targetIndex + 1, messagesRemoved, { resolveBlobs: false }).messages;
|
|
5716
|
+
const removedRefs = extractDiscordAwarenessRefs(discarded);
|
|
5717
|
+
const sourceBranch = this.store.currentBranch().name;
|
|
5718
|
+
const targetBranch = opts.branchName ?? `rollback/${agentName}/${Date.now()}`;
|
|
5719
|
+
if (targetBranch === sourceBranch) {
|
|
5720
|
+
throw new OperatorActionError('invalid', 'Rollback branch name must differ from the active branch');
|
|
5721
|
+
}
|
|
5722
|
+
|
|
5723
|
+
// Gate + reserve the whole store (see reserveStoreForSurgery); held
|
|
5724
|
+
// until the switch has landed or been rolled back.
|
|
5725
|
+
const release = this.reserveStoreForSurgery('roll back', agentName);
|
|
5726
|
+
let branchName: string;
|
|
5727
|
+
try {
|
|
5728
|
+
// Prepare the external side effect before switching Chronicle. If the
|
|
5729
|
+
// process dies after the switch but before activate(), startup
|
|
5730
|
+
// promotes this batch by matching targetBranch to the active branch.
|
|
5731
|
+
// If the fork/switch itself throws, the batch is retired so it cannot
|
|
5732
|
+
// arm a later branch that happens to reuse the name.
|
|
5733
|
+
const markerBatch = this.discordAwarenessOutbox?.prepare({
|
|
5734
|
+
agentName,
|
|
5735
|
+
sourceBranch,
|
|
5736
|
+
targetBranch,
|
|
5737
|
+
refs: removedRefs,
|
|
5738
|
+
emoji: this.discordAwarenessEmoji,
|
|
5739
|
+
}) ?? null;
|
|
5740
|
+
|
|
5741
|
+
try {
|
|
5742
|
+
branchName = cm.branchAt(tailMessageId as MessageId, targetBranch);
|
|
5743
|
+
await cm.switchBranch(branchName);
|
|
5744
|
+
} catch (error) {
|
|
5745
|
+
// switchBranch moves the chronicle branch BEFORE awaiting strategy
|
|
5746
|
+
// initialization, so a rejection here can leave the store on the
|
|
5747
|
+
// new branch with an uninitialized strategy. Retire the batch and
|
|
5748
|
+
// go back to the source; keep the branch for diagnosis.
|
|
5749
|
+
if (markerBatch) this.discordAwarenessOutbox!.discard(markerBatch.id);
|
|
5750
|
+
const restored = await this.restoreSourceBranch(cm, sourceBranch, targetBranch, agentName);
|
|
5751
|
+
throw new OperatorActionError(
|
|
5752
|
+
'failed',
|
|
5753
|
+
`Rollback failed; ${restored}: ${error instanceof Error ? error.message : String(error)}`,
|
|
5754
|
+
{ cause: error },
|
|
5755
|
+
);
|
|
5756
|
+
}
|
|
5757
|
+
if (markerBatch) this.discordAwarenessOutbox!.activate(markerBatch.id);
|
|
5758
|
+
await this.syncDiscordAwarenessMarkers();
|
|
5759
|
+
this.materializeConfigMountAfterBranchSwitch();
|
|
5760
|
+
} finally {
|
|
5761
|
+
release();
|
|
5762
|
+
}
|
|
5763
|
+
|
|
5764
|
+
console.error(
|
|
5765
|
+
`[operator] rollback agent=${agentName} to=${tailMessageId}` +
|
|
5766
|
+
`${tailMessageId !== opts.messageId ? ` (requested ${opts.messageId}, snapped to end of body group)` : ''}` +
|
|
5767
|
+
` removed=${messagesRemoved} branch=${branchName} by=${describeRequester(opts.requester)}`,
|
|
5768
|
+
);
|
|
5769
|
+
this.recordOperatorAction({
|
|
5770
|
+
...logBase,
|
|
5771
|
+
result: {
|
|
5772
|
+
sourceBranch,
|
|
5773
|
+
targetBranch: branchName,
|
|
5774
|
+
tailMessageId,
|
|
5775
|
+
messagesRemoved,
|
|
5776
|
+
discordRefs: removedRefs.length,
|
|
5777
|
+
},
|
|
5778
|
+
});
|
|
5779
|
+
return {
|
|
5780
|
+
agentName,
|
|
5781
|
+
sourceBranch,
|
|
5782
|
+
targetBranch: branchName,
|
|
5783
|
+
messagesRemoved,
|
|
5784
|
+
tailMessageId,
|
|
5785
|
+
removedRefs,
|
|
5786
|
+
lastVisible: await this.lastVisiblePreview(agentName),
|
|
5787
|
+
};
|
|
5788
|
+
} catch (error) {
|
|
5789
|
+
this.recordOperatorAction({ ...logBase, error: error instanceof Error ? error.message : String(error) });
|
|
5790
|
+
throw error;
|
|
5791
|
+
}
|
|
5792
|
+
}
|
|
5793
|
+
|
|
5794
|
+
/**
|
|
5795
|
+
* Suppress specific messages from the live context without touching the
|
|
5796
|
+
* source branch: fork at the current head (`suppress/<agent>/<ts>`), redact
|
|
5797
|
+
* the messages on the fork, switch to it. Shards of a body group are always
|
|
5798
|
+
* removed together. Discord originals get awareness markers via the outbox;
|
|
5799
|
+
* the batch is `explicit`-activated only after every removal succeeded, and
|
|
5800
|
+
* a crash mid-way is finished at next boot (resumePreparedDiscordSuppressions).
|
|
5801
|
+
*
|
|
5802
|
+
* Not retroactive over derived state: a message already folded into an
|
|
5803
|
+
* autobiographical summary stays in that summary — roll back to before it
|
|
5804
|
+
* entered if that matters.
|
|
5805
|
+
*/
|
|
5806
|
+
async suppressMessages(
|
|
5807
|
+
agentName: string,
|
|
5808
|
+
opts: {
|
|
5809
|
+
messageIds: string[];
|
|
5810
|
+
requester?: OperatorRequester;
|
|
5811
|
+
note?: string;
|
|
5812
|
+
/** Branch name for the fork (default `suppress/<agent>/<ts>`). */
|
|
5813
|
+
branchName?: string;
|
|
5814
|
+
},
|
|
5815
|
+
): Promise<{
|
|
5816
|
+
agentName: string;
|
|
5817
|
+
sourceBranch: string;
|
|
5818
|
+
targetBranch: string;
|
|
5819
|
+
messagesRemoved: number;
|
|
5820
|
+
removedIds: string[];
|
|
5821
|
+
removedRefs: Array<{ serverId: string; channelId: string; messageId: string }>;
|
|
5822
|
+
lastVisible: { participant?: string; role?: string; preview?: string } | null;
|
|
5823
|
+
}> {
|
|
5824
|
+
const requestedIds = [...new Set(opts.messageIds.map(String))];
|
|
5825
|
+
const logBase: OperatorLogInput = {
|
|
5826
|
+
kind: 'suppress',
|
|
5827
|
+
agent: agentName,
|
|
5828
|
+
...(opts.requester ? { requester: opts.requester } : {}),
|
|
5829
|
+
...(opts.note ? { note: opts.note } : {}),
|
|
5830
|
+
params: { messageIds: requestedIds },
|
|
5831
|
+
};
|
|
5832
|
+
try {
|
|
5833
|
+
const agent = this.agents.get(agentName);
|
|
5834
|
+
if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${agentName}`);
|
|
5835
|
+
if (requestedIds.length === 0) throw new OperatorActionError('invalid', 'No message ids given');
|
|
5836
|
+
const cm = agent.getContextManager();
|
|
5837
|
+
const total = cm.getMessageCount();
|
|
5838
|
+
|
|
5839
|
+
const located = locateMessageIndices(cm, new Set(requestedIds));
|
|
5840
|
+
const missing = requestedIds.filter((id) => !located.has(id));
|
|
5841
|
+
if (missing.length > 0) {
|
|
5842
|
+
throw new OperatorActionError(
|
|
5843
|
+
'unknown-message',
|
|
5844
|
+
`Not on the active branch: ${missing.slice(0, 5).join(', ')}${missing.length > 5 ? ` (+${missing.length - 5})` : ''}`,
|
|
5845
|
+
);
|
|
5846
|
+
}
|
|
5847
|
+
|
|
5848
|
+
// Build removal intervals (index-ordered, newest first). A shard expands
|
|
5849
|
+
// to its whole body group — chronicle refuses to bisect one — and a
|
|
5850
|
+
// group interval is always removed as a RANGE: `removeMessage` refuses
|
|
5851
|
+
// any sharded message even when the group is down to one shard.
|
|
5852
|
+
const intervals = new Map<string, ReturnType<typeof bodyGroupRun>>();
|
|
5853
|
+
for (const id of requestedIds) {
|
|
5854
|
+
const run = bodyGroupRun(cm, located.get(id)!.index);
|
|
5855
|
+
intervals.set(`${run.from}-${run.to}`, run);
|
|
5856
|
+
}
|
|
5857
|
+
const ordered = [...intervals.values()].sort((a, b) => b.from - a.from);
|
|
5858
|
+
const messagesRemoved = ordered.reduce((n, iv) => n + (iv.to - iv.from + 1), 0);
|
|
5859
|
+
if (messagesRemoved >= total) {
|
|
5860
|
+
throw new OperatorActionError('invalid', `Cannot suppress all ${total} message(s) — at least one must remain`);
|
|
5861
|
+
}
|
|
5862
|
+
const targeted = ordered.flatMap((iv) =>
|
|
5863
|
+
cm.getMessageWindow(iv.from, iv.to - iv.from + 1, { resolveBlobs: false }).messages,
|
|
5864
|
+
);
|
|
5865
|
+
const removedIds = targeted.map((m) => String(m.id));
|
|
5866
|
+
const removedRefs = extractDiscordAwarenessRefs(targeted);
|
|
5867
|
+
|
|
5868
|
+
const sourceBranch = this.store.currentBranch().name;
|
|
5869
|
+
const targetBranch = opts.branchName ?? `suppress/${agentName}/${Date.now()}`;
|
|
5870
|
+
if (targetBranch === sourceBranch) {
|
|
5871
|
+
throw new OperatorActionError('invalid', 'Suppression branch name must differ from the active branch');
|
|
5872
|
+
}
|
|
5873
|
+
// Gate + reserve the whole store (see reserveStoreForSurgery); held
|
|
5874
|
+
// until the fork is fully redacted or the source is restored.
|
|
5875
|
+
const release = this.reserveStoreForSurgery('suppress', agentName);
|
|
5876
|
+
let createdBranch: string;
|
|
5877
|
+
try {
|
|
5878
|
+
const markerBatch = this.discordAwarenessOutbox?.prepare({
|
|
5879
|
+
agentName,
|
|
5880
|
+
sourceBranch,
|
|
5881
|
+
targetBranch,
|
|
5882
|
+
refs: removedRefs,
|
|
5883
|
+
emoji: this.discordAwarenessEmoji,
|
|
5884
|
+
// Seeing targetBranch active does not prove the removals finished;
|
|
5885
|
+
// only this operation activates the batch, after the last redaction.
|
|
5886
|
+
activationPolicy: 'explicit',
|
|
5887
|
+
suppressionIntervals: ordered.map((iv) => ({ fromId: iv.fromId, toId: iv.toId })),
|
|
5888
|
+
}) ?? null;
|
|
5889
|
+
// Anything short of activation retires the batch: a prepared explicit
|
|
5890
|
+
// batch re-arms through preparedSuppressionsForBranch the moment its
|
|
5891
|
+
// target branch (or a descendant) becomes active — e.g. an operator
|
|
5892
|
+
// opening the failed fork to look — and an un-completable resume
|
|
5893
|
+
// there aborts framework start.
|
|
5894
|
+
const retireBatch = (): void => {
|
|
5895
|
+
if (markerBatch) this.discordAwarenessOutbox!.discard(markerBatch.id);
|
|
5896
|
+
};
|
|
5897
|
+
|
|
5898
|
+
try {
|
|
5899
|
+
createdBranch = await cm.fork(targetBranch);
|
|
5900
|
+
} catch (error) {
|
|
5901
|
+
// fork() switches the chronicle branch before awaiting strategy
|
|
5902
|
+
// initialization; a rejection can leave the store on the new
|
|
5903
|
+
// branch. Retire the batch and go back to the source.
|
|
5904
|
+
retireBatch();
|
|
5905
|
+
const restored = await this.restoreSourceBranch(cm, sourceBranch, targetBranch, agentName);
|
|
5906
|
+
throw new OperatorActionError(
|
|
5907
|
+
'failed',
|
|
5908
|
+
`Suppression failed creating ${targetBranch}; ${restored}: ${error instanceof Error ? error.message : String(error)}`,
|
|
5909
|
+
{ cause: error },
|
|
5910
|
+
);
|
|
5911
|
+
}
|
|
5912
|
+
try {
|
|
5913
|
+
for (const iv of ordered) {
|
|
5914
|
+
if (iv.group || iv.fromId !== iv.toId) cm.removeMessages(iv.fromId as MessageId, iv.toId as MessageId);
|
|
5915
|
+
else cm.removeMessage(iv.fromId as MessageId);
|
|
5916
|
+
}
|
|
5917
|
+
if (markerBatch) this.discordAwarenessOutbox!.activate(markerBatch.id);
|
|
5918
|
+
} catch (error) {
|
|
5919
|
+
// A partially suppressed branch is not safe to serve from. Keep it
|
|
5920
|
+
// for diagnosis, put the agent back on the untouched source — and
|
|
5921
|
+
// say honestly whether that restore happened.
|
|
5922
|
+
retireBatch();
|
|
5923
|
+
const restored = await this.restoreSourceBranch(cm, sourceBranch, createdBranch, agentName);
|
|
5924
|
+
throw new OperatorActionError(
|
|
5925
|
+
'failed',
|
|
5926
|
+
`Suppression failed on ${createdBranch}; ${restored}: ${error instanceof Error ? error.message : String(error)}`,
|
|
5927
|
+
{ cause: error },
|
|
5928
|
+
);
|
|
5929
|
+
}
|
|
5930
|
+
await this.syncDiscordAwarenessMarkers();
|
|
5931
|
+
this.materializeConfigMountAfterBranchSwitch();
|
|
5932
|
+
} finally {
|
|
5933
|
+
release();
|
|
5934
|
+
}
|
|
5935
|
+
|
|
5936
|
+
console.error(
|
|
5937
|
+
`[operator] suppress agent=${agentName} removed=${messagesRemoved} branch=${createdBranch} ` +
|
|
5938
|
+
`by=${describeRequester(opts.requester)}`,
|
|
5939
|
+
);
|
|
5940
|
+
const loggedIds = capIds(removedIds);
|
|
5941
|
+
this.recordOperatorAction({
|
|
5942
|
+
...logBase,
|
|
5943
|
+
result: {
|
|
5944
|
+
sourceBranch,
|
|
5945
|
+
targetBranch: createdBranch,
|
|
5946
|
+
messagesRemoved,
|
|
5947
|
+
removedIds: loggedIds.ids,
|
|
5948
|
+
...(loggedIds.truncated ? { removedIdsTruncated: true } : {}),
|
|
5949
|
+
discordRefs: removedRefs.length,
|
|
5950
|
+
},
|
|
5951
|
+
});
|
|
5952
|
+
return {
|
|
5953
|
+
agentName,
|
|
5954
|
+
sourceBranch,
|
|
5955
|
+
targetBranch: createdBranch,
|
|
5956
|
+
messagesRemoved,
|
|
5957
|
+
removedIds,
|
|
5958
|
+
removedRefs,
|
|
5959
|
+
lastVisible: await this.lastVisiblePreview(agentName),
|
|
5960
|
+
};
|
|
5961
|
+
} catch (error) {
|
|
5962
|
+
this.recordOperatorAction({ ...logBase, error: error instanceof Error ? error.message : String(error) });
|
|
5963
|
+
throw error;
|
|
5964
|
+
}
|
|
5965
|
+
}
|
|
5966
|
+
|
|
5967
|
+
/** Re-materialize config files from the (new) active branch — fire and
|
|
5968
|
+
* forget; the gate picks changes up via mtime. Mirrors undoLastTurn. */
|
|
5969
|
+
private materializeConfigMountAfterBranchSwitch(): void {
|
|
5970
|
+
const ws = this.moduleRegistry.getModule('workspace');
|
|
5971
|
+
if (ws && 'materializeMount' in ws) {
|
|
5972
|
+
(ws as { materializeMount: (m: string) => Promise<void> })
|
|
5973
|
+
.materializeMount('_config')
|
|
5974
|
+
.catch(() => {});
|
|
5975
|
+
}
|
|
5976
|
+
}
|
|
5977
|
+
|
|
4113
5978
|
/**
|
|
4114
5979
|
* Check if undo/redo is available for an agent.
|
|
4115
5980
|
*/
|
|
@@ -4252,7 +6117,12 @@ export class AgentFramework {
|
|
|
4252
6117
|
!this.queue.isEmpty ||
|
|
4253
6118
|
// Direct inference requests (e.g. runEphemeralToCompletion) bypass the
|
|
4254
6119
|
// event queue — without this the loop can exit before they're drained.
|
|
4255
|
-
|
|
6120
|
+
// While quiesced, gate-parked requests are NOT progress and must not
|
|
6121
|
+
// block idle (only a budget restart can still run); identity-equivalent
|
|
6122
|
+
// to `.length > 0` when not quiesced.
|
|
6123
|
+
this.pendingRequests.some(
|
|
6124
|
+
(r) => !this.quiesced || isTurnContinuation(r.reason),
|
|
6125
|
+
) ||
|
|
4256
6126
|
this.activeStreams.size > 0 ||
|
|
4257
6127
|
Array.from(this.agents.values()).some((a) => a.state.status !== 'idle')
|
|
4258
6128
|
) {
|
|
@@ -4506,7 +6376,7 @@ export class AgentFramework {
|
|
|
4506
6376
|
const deferred = this.drainDeferredFor(agent.name);
|
|
4507
6377
|
if (deferred.length > 0) {
|
|
4508
6378
|
for (const msg of deferred) {
|
|
4509
|
-
agent.getContextManager().addMessage(msg.participant, msg.content, msg.metadata);
|
|
6379
|
+
agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id));
|
|
4510
6380
|
// Injection guards: tool blocks would corrupt the tool-cycle
|
|
4511
6381
|
// structure the membrane enforces, and a message named as the
|
|
4512
6382
|
// agent itself would render as an ASSISTANT turn on the wire
|
|
@@ -4526,6 +6396,10 @@ export class AgentFramework {
|
|
|
4526
6396
|
}
|
|
4527
6397
|
}
|
|
4528
6398
|
|
|
6399
|
+
// Deferred writes handed to the store above: sync, then drop them
|
|
6400
|
+
// from the durable recovery queue (no-op unless one is persisted).
|
|
6401
|
+
this.ackDeferredWrites();
|
|
6402
|
+
|
|
4529
6403
|
// A newly injected CONVERSATIONAL message begins a new
|
|
4530
6404
|
// conversational round inside the same provider inference. Remember
|
|
4531
6405
|
// that boundary for driveStream: an explicit send in the preceding
|
|
@@ -5513,7 +7387,7 @@ export class AgentFramework {
|
|
|
5513
7387
|
|
|
5514
7388
|
const providerCooldown = this.providerAccelerationCooldowns?.get(agentName);
|
|
5515
7389
|
if (providerCooldown) {
|
|
5516
|
-
if (now < providerCooldown.until) {
|
|
7390
|
+
if (now < providerCooldown.until || this.extendHostProviderHold(agentName, providerCooldown)) {
|
|
5517
7391
|
for (const req of requests) {
|
|
5518
7392
|
if (!providerCooldown.heldRequests.some((r) => this.sameInferenceRequest(r, req))) {
|
|
5519
7393
|
providerCooldown.heldRequests.push(req);
|
|
@@ -5549,16 +7423,61 @@ export class AgentFramework {
|
|
|
5549
7423
|
// This mirrors how the restart has always overwritten activeStreams
|
|
5550
7424
|
// rather than waiting for the old stream's teardown.
|
|
5551
7425
|
const budgetRestart = requests.find((r) => r.reason === 'context_budget_restart');
|
|
7426
|
+
|
|
7427
|
+
// Host quiesce (issue #122): park wakes instead of starting turns —
|
|
7428
|
+
// the requeue mirrors the busy path below, so requests survive and
|
|
7429
|
+
// fire at resume. A context-budget restart passes through: it
|
|
7430
|
+
// CONTINUES a turn whose token is held (see the deadlock note below),
|
|
7431
|
+
// and quiesce's drain is exactly the phase where such a turn must be
|
|
7432
|
+
// allowed to finish. Parked requests are coalesced per reason (newest
|
|
7433
|
+
// kept): heartbeat/gate wakes arrive once per tick and nothing else
|
|
7434
|
+
// bounds a multi-day maintenance window's accumulation.
|
|
7435
|
+
if (this.quiesced) {
|
|
7436
|
+
// Continuations of a held turn (budget restart, non-streaming tool
|
|
7437
|
+
// results) pass; everything else parks.
|
|
7438
|
+
const passThrough = requests.filter((r) => isTurnContinuation(r.reason));
|
|
7439
|
+
const parked = requests.filter((r) => !isTurnContinuation(r.reason));
|
|
7440
|
+
// Coalesce per (reason, addressed): every channel wake shares one
|
|
7441
|
+
// reason, so keying on reason alone would let an ambient message
|
|
7442
|
+
// parked later evict a DM/mention parked earlier — and the
|
|
7443
|
+
// addressed-outranks-ambient trigger selection below would then
|
|
7444
|
+
// route the resumed turn into the wrong channel.
|
|
7445
|
+
const newestByKey = new Map<string, InferenceRequest>();
|
|
7446
|
+
for (const request of parked) {
|
|
7447
|
+
const key = `${request.reason}|${request.addressed ? 'a' : 'n'}`;
|
|
7448
|
+
const prev = newestByKey.get(key);
|
|
7449
|
+
if (!prev || request.timestamp >= prev.timestamp) {
|
|
7450
|
+
newestByKey.set(key, request);
|
|
7451
|
+
}
|
|
7452
|
+
}
|
|
7453
|
+
this.pendingRequests.push(...newestByKey.values());
|
|
7454
|
+
if (passThrough.length === 0) continue;
|
|
7455
|
+
// The continuation continues its held turn; its parked siblings must
|
|
7456
|
+
// NOT be consumed by that turn — they were re-parked above, so narrow
|
|
7457
|
+
// the batch to the continuation(s) alone before the trigger selection.
|
|
7458
|
+
requests.length = 0;
|
|
7459
|
+
requests.push(...passThrough);
|
|
7460
|
+
}
|
|
7461
|
+
|
|
5552
7462
|
const turnAlive = !budgetRestart && this.activeTurnTokens.has(agentName);
|
|
5553
7463
|
const providerGate = this.providerGates?.get(agentName);
|
|
5554
7464
|
// A primary can own provider admission while yielding to an already in-flight
|
|
5555
7465
|
// auxiliary call. Keep this agent's later wakes queued, but do not block the
|
|
5556
7466
|
// framework event loop or other residents while that auxiliary call settles.
|
|
5557
7467
|
const providerPrimaryWaiting = (providerGate?.primaryDepth ?? 0) > 0 && !this.activeTurnTokens.has(agentName);
|
|
5558
|
-
|
|
5559
|
-
|
|
7468
|
+
// A live surgery (rollback/suppress) holds the whole store: no turn may
|
|
7469
|
+
// start for ANY agent — including one registered after the reservation's
|
|
7470
|
+
// token snapshot — until the switch has landed or been rolled back.
|
|
7471
|
+
// Truthiness, not `!== null`: prototype-built harnesses (and any field
|
|
7472
|
+
// added later) leave the slot undefined, which must read as "not held".
|
|
7473
|
+
const surgeryHeld = !!this.surgeryHold;
|
|
7474
|
+
if (surgeryHeld || providerPrimaryWaiting || turnAlive || agent.state.status === 'inferring' || agent.state.status === 'streaming' || agent.state.status === 'waiting_for_tools') {
|
|
7475
|
+
// Re-queue requests, but warn if they've been pending too long.
|
|
7476
|
+
// Suppressed while quiesced: a drain-phase busy requeue is expected,
|
|
7477
|
+
// not a wedge tell.
|
|
5560
7478
|
const oldest = Math.min(...requests.map(r => r.timestamp));
|
|
5561
7479
|
if (
|
|
7480
|
+
!this.quiesced &&
|
|
5562
7481
|
now - oldest > STALE_REQUEST_MS &&
|
|
5563
7482
|
(this.staleWarnAt.get(agentName) ?? 0) < now - 60_000
|
|
5564
7483
|
) {
|
|
@@ -5571,9 +7490,11 @@ export class AgentFramework {
|
|
|
5571
7490
|
// read 'idle' while a turn's teardown is pending, and a LEAKED turn
|
|
5572
7491
|
// token would look exactly like this — permanently requeued wakes.
|
|
5573
7492
|
// This line is the wedge's tell (idle+turn-alive, forever).
|
|
5574
|
-
const shownStatus =
|
|
5575
|
-
?
|
|
5576
|
-
: agent.state.status
|
|
7493
|
+
const shownStatus = surgeryHeld && agent.state.status === 'idle'
|
|
7494
|
+
? `idle+surgery(${this.surgeryHold!.verb})`
|
|
7495
|
+
: turnAlive && agent.state.status === 'idle'
|
|
7496
|
+
? 'idle+turn-alive'
|
|
7497
|
+
: agent.state.status;
|
|
5577
7498
|
this.emitTrace({
|
|
5578
7499
|
type: 'inference:request_stale',
|
|
5579
7500
|
agentName,
|
|
@@ -6149,6 +8070,20 @@ export class AgentFramework {
|
|
|
6149
8070
|
console.error(`[provider-admission] ${agent.name}: turn-alive while parked — wake requeued, not started`);
|
|
6150
8071
|
return;
|
|
6151
8072
|
}
|
|
8073
|
+
// Same gap, second hazard (#122): the host may have QUIESCED while
|
|
8074
|
+
// this wake was parked. The scheduler's quiesce gate ran before the
|
|
8075
|
+
// park; starting here would bypass it and run a turn against a
|
|
8076
|
+
// context the operator believes frozen. Give admission back and
|
|
8077
|
+
// requeue — the scheduler re-parks it (coalesced) until resume.
|
|
8078
|
+
// Continuations of a held turn pass, as they do at the gate.
|
|
8079
|
+
if (this.quiesced && !isTurnContinuation(trigger?.reason ?? '')) {
|
|
8080
|
+
this.releasePrimaryProviderGate(agent.name);
|
|
8081
|
+
this.pendingRequests.push(trigger ?? {
|
|
8082
|
+
agentName: agent.name, reason: 'provider-admission:requeue', source: 'scheduler', timestamp: Date.now(),
|
|
8083
|
+
});
|
|
8084
|
+
console.error(`[provider-admission] ${agent.name}: host quiesced while parked — wake requeued, not started`);
|
|
8085
|
+
return;
|
|
8086
|
+
}
|
|
6152
8087
|
await this.startAgentStream(agent, trigger, attempt, true);
|
|
6153
8088
|
}).catch((error) => {
|
|
6154
8089
|
this.releasePrimaryProviderGate(agent.name);
|
|
@@ -6230,7 +8165,9 @@ export class AgentFramework {
|
|
|
6230
8165
|
// poison message — or a transient store-write failure — must not
|
|
6231
8166
|
// abort the turn or drop the messages behind it in the queue.
|
|
6232
8167
|
try {
|
|
6233
|
-
const id = agent.getContextManager().addMessage(
|
|
8168
|
+
const id = agent.getContextManager().addMessage(
|
|
8169
|
+
msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id),
|
|
8170
|
+
);
|
|
6234
8171
|
this.emitTrace({ type: 'message:added', messageId: id, source: 'deferred-flush:turn-start' });
|
|
6235
8172
|
} catch (err) {
|
|
6236
8173
|
console.error(
|
|
@@ -6240,6 +8177,7 @@ export class AgentFramework {
|
|
|
6240
8177
|
);
|
|
6241
8178
|
}
|
|
6242
8179
|
}
|
|
8180
|
+
this.ackDeferredWrites();
|
|
6243
8181
|
}
|
|
6244
8182
|
}
|
|
6245
8183
|
|
|
@@ -7415,6 +9353,41 @@ export class AgentFramework {
|
|
|
7415
9353
|
this.abortAgentScript(agent.name, 'stream error');
|
|
7416
9354
|
agent.reset();
|
|
7417
9355
|
|
|
9356
|
+
// A quiesce-abandoned cancel may surface as `error` instead of
|
|
9357
|
+
// `aborted` depending on how the stream implementation reports
|
|
9358
|
+
// the cancellation. Same contract as the aborted branch: settle
|
|
9359
|
+
// honestly, no errorPolicy retry (which would relaunch inference
|
|
9360
|
+
// mid-maintenance-window), no inference:exhausted (which feeds
|
|
9361
|
+
// the failure streak / hard-down / poison-history accounting).
|
|
9362
|
+
{
|
|
9363
|
+
const cancelKey = `${agent.name}:${myStreamId}`;
|
|
9364
|
+
if (this.frameworkCancelledStreams.get(cancelKey) === 'quiesce_abandoned') {
|
|
9365
|
+
this.frameworkCancelledStreams.delete(cancelKey);
|
|
9366
|
+
// Same terminal as the `aborted` twin: §10.5 lifecycle reads
|
|
9367
|
+
// 'aborted' (not 'completed'), and the gate is released only
|
|
9368
|
+
// for THIS agent instance — a name re-registered meanwhile
|
|
9369
|
+
// (ephemeral disposal, conversation replacement) owns its own
|
|
9370
|
+
// gate liveness. The inference-log terminal was already
|
|
9371
|
+
// written at the top of this case (`Stream error`).
|
|
9372
|
+
lifecyclePhase = 'aborted';
|
|
9373
|
+
this.settleAgent(agent.name, {
|
|
9374
|
+
stopReason: 'exhausted',
|
|
9375
|
+
speech: '',
|
|
9376
|
+
error: 'Turn abandoned by operator quiesce',
|
|
9377
|
+
});
|
|
9378
|
+
this.emitTrace({
|
|
9379
|
+
type: 'inference:aborted',
|
|
9380
|
+
agentName: agent.name,
|
|
9381
|
+
durationMs,
|
|
9382
|
+
reason: 'quiesce_abandoned',
|
|
9383
|
+
});
|
|
9384
|
+
if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) {
|
|
9385
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
9386
|
+
}
|
|
9387
|
+
break;
|
|
9388
|
+
}
|
|
9389
|
+
}
|
|
9390
|
+
|
|
7418
9391
|
if (ownsProviderGate && this.holdProviderAcceleration(agent, err, trigger)) {
|
|
7419
9392
|
lifecyclePhase = 'failed';
|
|
7420
9393
|
if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) this.eventGate?.onInferenceEnded(agent.name);
|
|
@@ -7470,6 +9443,30 @@ export class AgentFramework {
|
|
|
7470
9443
|
} else if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) {
|
|
7471
9444
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
7472
9445
|
}
|
|
9446
|
+
if (cancelKind === 'quiesce_abandoned') {
|
|
9447
|
+
// Operator-initiated cancel during a quiesce drain. Settle
|
|
9448
|
+
// the turn honestly — deliberately NO inference:exhausted:
|
|
9449
|
+
// that trace centrally drives the consecutive-failure
|
|
9450
|
+
// streak, hard-down ops alerts, and the poison-history
|
|
9451
|
+
// breaker, none of which an operator cancel represents.
|
|
9452
|
+
// inference:aborted carries the observability instead.
|
|
9453
|
+
if (agent.streamId === myStreamId) {
|
|
9454
|
+
this.abortAgentScript(agent.name, 'turn abandoned by quiesce');
|
|
9455
|
+
agent.reset();
|
|
9456
|
+
this.settleAgent(agent.name, {
|
|
9457
|
+
stopReason: 'exhausted',
|
|
9458
|
+
speech: '',
|
|
9459
|
+
error: 'Turn abandoned by operator quiesce',
|
|
9460
|
+
});
|
|
9461
|
+
}
|
|
9462
|
+
this.emitTrace({
|
|
9463
|
+
type: 'inference:aborted',
|
|
9464
|
+
agentName: agent.name,
|
|
9465
|
+
durationMs: Date.now() - startTime,
|
|
9466
|
+
reason: 'quiesce_abandoned',
|
|
9467
|
+
});
|
|
9468
|
+
return;
|
|
9469
|
+
}
|
|
7473
9470
|
// endTurn IS a logical turn end — earlier rounds may have
|
|
7474
9471
|
// live-routed prose (narrate → skip_reply is a real shape),
|
|
7475
9472
|
// so settle the delivery chain and drop the receipt. A
|
|
@@ -7761,9 +9758,12 @@ export class AgentFramework {
|
|
|
7761
9758
|
if (frameReachedTerminal && this.deferredMessages.length > 0 && this.pendingAssistantBlocks.size === 0) {
|
|
7762
9759
|
const deferred = this.drainDeferredFor(agent.name);
|
|
7763
9760
|
for (const msg of deferred) {
|
|
7764
|
-
this.addMessage(msg.participant, msg.content, msg.metadata,
|
|
7765
|
-
|
|
9761
|
+
this.addMessage(msg.participant, msg.content, msg.metadata, {
|
|
9762
|
+
deferredWriteId: msg.id,
|
|
9763
|
+
...(msg.forAgent ? { forAgent: msg.forAgent } : {}),
|
|
9764
|
+
});
|
|
7766
9765
|
}
|
|
9766
|
+
this.ackDeferredWrites();
|
|
7767
9767
|
}
|
|
7768
9768
|
}
|
|
7769
9769
|
}
|
|
@@ -7834,6 +9834,11 @@ export class AgentFramework {
|
|
|
7834
9834
|
// Idle AND no turn alive: status reads 'idle' from dequeue until the
|
|
7835
9835
|
// stream registers, and again while a turn's teardown is pending
|
|
7836
9836
|
// (the scheduler's own busy test, 'idle+turn-alive').
|
|
9837
|
+
if (this.surgeryHold) {
|
|
9838
|
+
throw new Error(
|
|
9839
|
+
`puppet refused: the store is under live ${this.surgeryHold.verb} for ${this.surgeryHold.agentName} — retry when it completes`,
|
|
9840
|
+
);
|
|
9841
|
+
}
|
|
7837
9842
|
if (agent.state.status !== 'idle' || this.activeTurnTokens.has(agentName)) {
|
|
7838
9843
|
const shown = agent.state.status === 'idle' ? 'idle+turn-alive' : agent.state.status;
|
|
7839
9844
|
throw new Error(
|
|
@@ -7909,8 +9914,8 @@ export class AgentFramework {
|
|
|
7909
9914
|
cm.addMessage('user', blocks);
|
|
7910
9915
|
} else {
|
|
7911
9916
|
this.deferredMessages.push(
|
|
7912
|
-
{ participant: agentName, content: toolUse, forAgent: agentName },
|
|
7913
|
-
{ participant: 'user', content: blocks, forAgent: agentName },
|
|
9917
|
+
{ id: randomUUID(), seq: ++this.deferredSeq, participant: agentName, content: toolUse, forAgent: agentName },
|
|
9918
|
+
{ id: randomUUID(), seq: ++this.deferredSeq, participant: 'user', content: blocks, forAgent: agentName },
|
|
7914
9919
|
);
|
|
7915
9920
|
}
|
|
7916
9921
|
|
|
@@ -7940,9 +9945,12 @@ export class AgentFramework {
|
|
|
7940
9945
|
if (!this.activeTurnTokens.has(agentName)
|
|
7941
9946
|
&& this.deferredMessages.length > 0 && this.pendingAssistantBlocks.size === 0) {
|
|
7942
9947
|
for (const msg of this.drainDeferredFor(agentName)) {
|
|
7943
|
-
this.addMessage(msg.participant, msg.content, msg.metadata,
|
|
7944
|
-
|
|
9948
|
+
this.addMessage(msg.participant, msg.content, msg.metadata, {
|
|
9949
|
+
deferredWriteId: msg.id,
|
|
9950
|
+
...(msg.forAgent ? { forAgent: msg.forAgent } : {}),
|
|
9951
|
+
});
|
|
7945
9952
|
}
|
|
9953
|
+
this.ackDeferredWrites();
|
|
7946
9954
|
}
|
|
7947
9955
|
}
|
|
7948
9956
|
}
|
|
@@ -9146,6 +11154,14 @@ export class AgentFramework {
|
|
|
9146
11154
|
* this is reachable from timer callbacks that may outlive an agent).
|
|
9147
11155
|
*/
|
|
9148
11156
|
forAgent?: string;
|
|
11157
|
+
/**
|
|
11158
|
+
* Set when the message is being flushed FROM the durable deferred
|
|
11159
|
+
* queue: it is the entry's one durable identity. A write stamps it
|
|
11160
|
+
* into `metadata.deferredWriteId` (boot dedup); a re-deferral moves
|
|
11161
|
+
* the same logical entry back to pending under the same id instead of
|
|
11162
|
+
* minting a second, independently replayable one.
|
|
11163
|
+
*/
|
|
11164
|
+
deferredWriteId?: string;
|
|
9149
11165
|
}
|
|
9150
11166
|
): MessageId {
|
|
9151
11167
|
// Route to the named agent, else the primary (not ephemeral subagents).
|
|
@@ -9181,6 +11197,10 @@ export class AgentFramework {
|
|
|
9181
11197
|
// their own wake), at the agent's next tool boundary (where they are
|
|
9182
11198
|
// ALSO injected into the live stream — hear-while-acting), or in
|
|
9183
11199
|
// driveStream's finally when the turn ends.
|
|
11200
|
+
// Quiesce (issue #122): the window exists to run compression/refold
|
|
11201
|
+
// against a FROZEN context — module events and api message.send must not
|
|
11202
|
+
// append mid-surgery. Deferred here, flushed by resume(). tool_result
|
|
11203
|
+
// still lands (a draining turn's continuation depends on it).
|
|
9184
11204
|
const hasToolResult = content.some(b => b.type === 'tool_result');
|
|
9185
11205
|
// Explicitly-targeted deliveries scope the tool-cycle check to the
|
|
9186
11206
|
// target (pendingAssistantBlocks is keyed by agent); the default path
|
|
@@ -9191,15 +11211,30 @@ export class AgentFramework {
|
|
|
9191
11211
|
: this.pendingAssistantBlocks.size > 0;
|
|
9192
11212
|
if (
|
|
9193
11213
|
!hasToolResult &&
|
|
9194
|
-
(
|
|
11214
|
+
(this.quiesced ||
|
|
11215
|
+
midToolCycle ||
|
|
9195
11216
|
this.activeTurnTokens.has(agent.name) ||
|
|
9196
11217
|
this.activeStreams.has(agent.name))
|
|
9197
11218
|
) {
|
|
9198
|
-
|
|
11219
|
+
// A re-deferral of a drained entry keeps its id and leaves the un-acked
|
|
11220
|
+
// set: one logical message, one durable identity — never two entries.
|
|
11221
|
+
const id = opts?.deferredWriteId ?? randomUUID();
|
|
11222
|
+
let seq: number | undefined;
|
|
11223
|
+
if (opts?.deferredWriteId) {
|
|
11224
|
+
const prior = this.unackedDeferredWrites.find((m) => m.id === id);
|
|
11225
|
+
seq = prior?.seq;
|
|
11226
|
+
this.unackedDeferredWrites = this.unackedDeferredWrites.filter((m) => m.id !== id);
|
|
11227
|
+
}
|
|
11228
|
+
this.deferredMessages.push({ id, seq: seq ?? ++this.deferredSeq, participant, content, metadata, forAgent: opts?.forAgent });
|
|
11229
|
+
if (this.quiesced || this.deferredWritesPersisted) this.persistDeferredWrites();
|
|
9199
11230
|
return '' as MessageId; // Deferred — flushed at the target's next boundary
|
|
9200
11231
|
}
|
|
9201
11232
|
|
|
9202
|
-
return agent.getContextManager().addMessage(
|
|
11233
|
+
return agent.getContextManager().addMessage(
|
|
11234
|
+
participant,
|
|
11235
|
+
content,
|
|
11236
|
+
opts?.deferredWriteId ? withDeferredWriteId(metadata, opts.deferredWriteId) : metadata,
|
|
11237
|
+
);
|
|
9203
11238
|
}
|
|
9204
11239
|
|
|
9205
11240
|
/**
|
|
@@ -9208,6 +11243,9 @@ export class AgentFramework {
|
|
|
9208
11243
|
* targets' messages queued for their own boundaries.
|
|
9209
11244
|
*/
|
|
9210
11245
|
private drainDeferredFor(agentName: string): Array<{
|
|
11246
|
+
/** Durable identity — every write of this entry must carry it
|
|
11247
|
+
* (`withDeferredWriteId` / `opts.deferredWriteId`). */
|
|
11248
|
+
id: string;
|
|
9211
11249
|
participant: string;
|
|
9212
11250
|
content: ContentBlock[];
|
|
9213
11251
|
metadata?: MessageMetadata;
|
|
@@ -9221,6 +11259,12 @@ export class AgentFramework {
|
|
|
9221
11259
|
(target === agentName ? mine : rest).push(msg);
|
|
9222
11260
|
}
|
|
9223
11261
|
this.deferredMessages = rest;
|
|
11262
|
+
mine.sort(bySeq);
|
|
11263
|
+
// The drained messages are about to be written by the caller; they stay
|
|
11264
|
+
// in the DURABLE queue (as un-acked) until the caller's
|
|
11265
|
+
// ackDeferredWrites() has synced the chronicle. Removing them here would
|
|
11266
|
+
// acknowledge before the write is durable.
|
|
11267
|
+
this.handOffDeferredWrites(mine);
|
|
9224
11268
|
return mine;
|
|
9225
11269
|
}
|
|
9226
11270
|
|
|
@@ -9370,6 +11414,8 @@ export class AgentFramework {
|
|
|
9370
11414
|
auxiliaryInFlight: gate?.auxiliaryInFlight ?? 0,
|
|
9371
11415
|
auxiliaryDeferred: gate?.deferredAuxiliary ?? 0,
|
|
9372
11416
|
cooldownUntil: cooldown?.until ?? null,
|
|
11417
|
+
cooldownReason: cooldown?.reason ?? null,
|
|
11418
|
+
hostHold: cooldown?.hostHoldError !== undefined,
|
|
9373
11419
|
heldRequests: cooldown?.heldRequests.length ?? 0,
|
|
9374
11420
|
lastRecovery: this.providerAccelerationLastRecovery?.get(name) ?? null,
|
|
9375
11421
|
};
|
|
@@ -10212,8 +12258,21 @@ export class AgentFramework {
|
|
|
10212
12258
|
for (const connection of this.mcplServerRegistry?.getAllServers() ?? []) {
|
|
10213
12259
|
// ready() can synchronously flush a nested list-change notification that
|
|
10214
12260
|
// installs a newer global generation. Never let this older completion
|
|
10215
|
-
// release any remaining server behind that newer gate.
|
|
12261
|
+
// release any remaining server behind that newer gate. (The quiesce
|
|
12262
|
+
// check sits AFTER this guard: readyControlPlane can also synchronously
|
|
12263
|
+
// flush a control event that installs a newer barrier.)
|
|
10216
12264
|
if (this.discordAwarenessBarrier !== null) return false;
|
|
12265
|
+
// Host quiesce (issue #122): this is the single point every barrier
|
|
12266
|
+
// completion and reconnect flow funnels through, so holding here keeps
|
|
12267
|
+
// data planes paused across awareness drains and reconnects for the
|
|
12268
|
+
// whole window. Control planes come up normally (host/command rides the
|
|
12269
|
+
// control plane, so resume stays deliverable); resume() performs the
|
|
12270
|
+
// real ready() flush through this same funnel once its feasibility gate
|
|
12271
|
+
// passes and the flag is cleared.
|
|
12272
|
+
if (this.quiesced) {
|
|
12273
|
+
connection.readyControlPlane();
|
|
12274
|
+
continue;
|
|
12275
|
+
}
|
|
10217
12276
|
connection.ready();
|
|
10218
12277
|
}
|
|
10219
12278
|
return this.discordAwarenessBarrier === null;
|