@animalabs/agent-framework 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -1
- package/dist/src/agent.d.ts +42 -0
- package/dist/src/agent.d.ts.map +1 -1
- package/dist/src/agent.js +94 -0
- package/dist/src/agent.js.map +1 -1
- package/dist/src/api/server.d.ts +9 -0
- package/dist/src/api/server.d.ts.map +1 -1
- package/dist/src/api/server.js +156 -0
- package/dist/src/api/server.js.map +1 -1
- package/dist/src/api/types.d.ts +15 -0
- package/dist/src/api/types.d.ts.map +1 -1
- package/dist/src/framework.d.ts +423 -5
- package/dist/src/framework.d.ts.map +1 -1
- package/dist/src/framework.js +1906 -102
- package/dist/src/framework.js.map +1 -1
- package/dist/src/gate/event-gate.d.ts +16 -0
- package/dist/src/gate/event-gate.d.ts.map +1 -1
- package/dist/src/gate/event-gate.js +41 -3
- package/dist/src/gate/event-gate.js.map +1 -1
- package/dist/src/index.d.ts +4 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +3 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/mcpl/channel-registry.d.ts +172 -0
- package/dist/src/mcpl/channel-registry.d.ts.map +1 -1
- package/dist/src/mcpl/channel-registry.js +444 -2
- package/dist/src/mcpl/channel-registry.js.map +1 -1
- package/dist/src/mcpl/server-connection.d.ts +9 -0
- package/dist/src/mcpl/server-connection.d.ts.map +1 -1
- package/dist/src/mcpl/server-connection.js +71 -13
- package/dist/src/mcpl/server-connection.js.map +1 -1
- package/dist/src/modules/history/index.d.ts +145 -0
- package/dist/src/modules/history/index.d.ts.map +1 -0
- package/dist/src/modules/history/index.js +1150 -0
- package/dist/src/modules/history/index.js.map +1 -0
- package/dist/src/modules/history/search-regex-worker.d.ts +23 -0
- package/dist/src/modules/history/search-regex-worker.d.ts.map +1 -0
- package/dist/src/modules/history/search-regex-worker.js +57 -0
- package/dist/src/modules/history/search-regex-worker.js.map +1 -0
- package/dist/src/modules/index.d.ts +1 -0
- package/dist/src/modules/index.d.ts.map +1 -1
- package/dist/src/modules/index.js +1 -0
- package/dist/src/modules/index.js.map +1 -1
- package/dist/src/modules/workspace/index.d.ts.map +1 -1
- package/dist/src/modules/workspace/index.js +42 -1
- package/dist/src/modules/workspace/index.js.map +1 -1
- package/dist/src/modules/workspace/types.d.ts +5 -0
- package/dist/src/modules/workspace/types.d.ts.map +1 -1
- package/dist/src/modules/workspace/types.js.map +1 -1
- package/dist/src/operator-log.d.ts +78 -0
- package/dist/src/operator-log.d.ts.map +1 -0
- package/dist/src/operator-log.js +102 -0
- package/dist/src/operator-log.js.map +1 -0
- package/dist/src/recovery/discord-awareness-outbox.d.ts +10 -0
- package/dist/src/recovery/discord-awareness-outbox.d.ts.map +1 -1
- package/dist/src/recovery/discord-awareness-outbox.js +20 -0
- package/dist/src/recovery/discord-awareness-outbox.js.map +1 -1
- package/dist/src/types/framework.d.ts +24 -0
- package/dist/src/types/framework.d.ts.map +1 -1
- package/dist/src/types/trace.d.ts +40 -0
- package/dist/src/types/trace.d.ts.map +1 -1
- package/dist/test/agent-runtime-settings.test.js +199 -1
- package/dist/test/agent-runtime-settings.test.js.map +1 -1
- package/dist/test/api-admin-gate.test.d.ts +2 -0
- package/dist/test/api-admin-gate.test.d.ts.map +1 -0
- package/dist/test/api-admin-gate.test.js +116 -0
- package/dist/test/api-admin-gate.test.js.map +1 -0
- package/dist/test/channel-label-history.test.d.ts +2 -0
- package/dist/test/channel-label-history.test.d.ts.map +1 -0
- package/dist/test/channel-label-history.test.js +424 -0
- package/dist/test/channel-label-history.test.js.map +1 -0
- package/dist/test/discord-awareness-outbox.test.js +16 -0
- package/dist/test/discord-awareness-outbox.test.js.map +1 -1
- package/dist/test/event-gate.test.js +44 -0
- package/dist/test/event-gate.test.js.map +1 -1
- package/dist/test/helpers/quiesce-crash-child.d.ts +2 -0
- package/dist/test/helpers/quiesce-crash-child.d.ts.map +1 -0
- package/dist/test/helpers/quiesce-crash-child.js +156 -0
- package/dist/test/helpers/quiesce-crash-child.js.map +1 -0
- package/dist/test/history-module-overview.test.d.ts +2 -0
- package/dist/test/history-module-overview.test.d.ts.map +1 -0
- package/dist/test/history-module-overview.test.js +711 -0
- package/dist/test/history-module-overview.test.js.map +1 -0
- package/dist/test/history-module.test.d.ts +2 -0
- package/dist/test/history-module.test.d.ts.map +1 -0
- package/dist/test/history-module.test.js +595 -0
- package/dist/test/history-module.test.js.map +1 -0
- package/dist/test/host-quiesce.test.d.ts +2 -0
- package/dist/test/host-quiesce.test.d.ts.map +1 -0
- package/dist/test/host-quiesce.test.js +863 -0
- package/dist/test/host-quiesce.test.js.map +1 -0
- package/dist/test/kv-unified-open-flight-retry.test.d.ts +2 -0
- package/dist/test/kv-unified-open-flight-retry.test.d.ts.map +1 -0
- package/dist/test/kv-unified-open-flight-retry.test.js +189 -0
- package/dist/test/kv-unified-open-flight-retry.test.js.map +1 -0
- package/dist/test/kv-unified-wiring.test.js +39 -0
- package/dist/test/kv-unified-wiring.test.js.map +1 -1
- package/dist/test/operator-surgery.test.d.ts +2 -0
- package/dist/test/operator-surgery.test.d.ts.map +1 -0
- package/dist/test/operator-surgery.test.js +398 -0
- package/dist/test/operator-surgery.test.js.map +1 -0
- package/dist/test/puppet-tool-call.test.js +118 -1
- package/dist/test/puppet-tool-call.test.js.map +1 -1
- package/dist/test/puppet-turn-reservation.test.d.ts +2 -0
- package/dist/test/puppet-turn-reservation.test.d.ts.map +1 -0
- package/dist/test/puppet-turn-reservation.test.js +139 -0
- package/dist/test/puppet-turn-reservation.test.js.map +1 -0
- package/dist/test/save-recent-image.test.js +29 -0
- package/dist/test/save-recent-image.test.js.map +1 -1
- package/dist/test/tool-result-spill.test.js +114 -0
- package/dist/test/tool-result-spill.test.js.map +1 -1
- package/dist/test/workspace-read-text.test.d.ts +2 -0
- package/dist/test/workspace-read-text.test.d.ts.map +1 -0
- package/dist/test/workspace-read-text.test.js +101 -0
- package/dist/test/workspace-read-text.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +3 -3
- package/src/agent.ts +110 -0
- package/src/api/server.ts +159 -2
- package/src/api/types.ts +15 -0
- package/src/framework.ts +2196 -115
- package/src/gate/event-gate.ts +47 -4
- package/src/index.ts +5 -1
- package/src/mcpl/channel-registry.ts +495 -2
- package/src/mcpl/server-connection.ts +73 -13
- package/src/modules/history/index.ts +1277 -0
- package/src/modules/history/search-regex-worker.ts +76 -0
- package/src/modules/index.ts +2 -0
- package/src/modules/workspace/index.ts +42 -1
- package/src/modules/workspace/types.ts +5 -0
- package/src/operator-log.ts +132 -0
- package/src/recovery/discord-awareness-outbox.ts +19 -0
- package/src/types/framework.ts +27 -0
- package/src/types/trace.ts +46 -0
package/src/framework.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { join } from 'node:path';
|
|
1
|
+
import { dirname, join } from 'node:path';
|
|
2
2
|
import { INLINE_WITHHELD_TEXT, classifyBlock, isInlineContradiction, referenceRegistry, referenceStubOrNull } from './mcpl/references.js';
|
|
3
3
|
import { ReferenceFetcher, DEFAULT_FETCH_MAX_BYTES, EAGER_FETCH_TIMEOUT_MS } from './mcpl/reference-fetcher.js';
|
|
4
|
-
import { appendFileSync, mkdirSync } from 'node:fs';
|
|
4
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
|
5
5
|
import { JsStore } from '@animalabs/chronicle';
|
|
6
6
|
import type { Membrane, ContentBlock, NormalizedRequest, YieldingStream, ToolResult as MembraneToolResult, ToolResultContentBlock } from '@animalabs/membrane';
|
|
7
7
|
import { MembraneError } from '@animalabs/membrane';
|
|
@@ -394,8 +394,50 @@ import {
|
|
|
394
394
|
defaultDiscordAwarenessOutboxPath,
|
|
395
395
|
extractDiscordAwarenessRefs,
|
|
396
396
|
} from './recovery/discord-awareness-outbox.js';
|
|
397
|
+
import {
|
|
398
|
+
OperatorActionError,
|
|
399
|
+
OperatorLog,
|
|
400
|
+
capIds,
|
|
401
|
+
defaultOperatorLogPath,
|
|
402
|
+
type OperatorLogEntry,
|
|
403
|
+
type OperatorLogInput,
|
|
404
|
+
type OperatorRequester,
|
|
405
|
+
} from './operator-log.js';
|
|
397
406
|
|
|
398
407
|
const FRAMEWORK_STATE_ID = 'framework/state';
|
|
408
|
+
/** Snapshot slot mirroring context writes deferred while quiesced (#122). */
|
|
409
|
+
const DEFERRED_WRITES_ID = 'framework/deferred-writes';
|
|
410
|
+
/** Above this serialized size the deferred-write mirror is skipped (kept in
|
|
411
|
+
* memory only) rather than rewriting a multi-MB snapshot per deferral. */
|
|
412
|
+
const DEFERRED_WRITES_PERSIST_CAP_BYTES = 4 * 1024 * 1024;
|
|
413
|
+
|
|
414
|
+
/** Requests that CONTINUE a turn whose token is already held — a context-
|
|
415
|
+
* budget restart, or the non-streaming path's tool-results round. The
|
|
416
|
+
* quiesce wake gate lets these through: the drain is exactly the phase
|
|
417
|
+
* where such a turn must be allowed to finish, and parking them would hold
|
|
418
|
+
* the token until the drain timed out (and `abandon` could not clear a turn
|
|
419
|
+
* with no stream). */
|
|
420
|
+
/** One entry of the deferred-write queue (see AgentFramework.deferredMessages). */
|
|
421
|
+
interface DeferredWrite {
|
|
422
|
+
id: string;
|
|
423
|
+
seq: number;
|
|
424
|
+
participant: string;
|
|
425
|
+
content: ContentBlock[];
|
|
426
|
+
metadata?: MessageMetadata;
|
|
427
|
+
forAgent?: string;
|
|
428
|
+
}
|
|
429
|
+
const bySeq = (a: { seq: number }, b: { seq: number }): number => a.seq - b.seq;
|
|
430
|
+
|
|
431
|
+
/** Stamp a deferred write's durable id into the metadata it is stored with
|
|
432
|
+
* (boot recovery dedups replays by it). Idempotent for an already-stamped
|
|
433
|
+
* message. */
|
|
434
|
+
function withDeferredWriteId(metadata: MessageMetadata | undefined, id: string): MessageMetadata {
|
|
435
|
+
return { ...(metadata ?? {}), deferredWriteId: id } as MessageMetadata;
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
function isTurnContinuation(reason: string): boolean {
|
|
439
|
+
return reason === 'context_budget_restart' || reason === 'tool_results_ready';
|
|
440
|
+
}
|
|
399
441
|
const CONVERSATION_ROUTER_STATE_ID = 'framework/conversation-router';
|
|
400
442
|
const INFERENCE_LOG_ID = 'framework/inference-log';
|
|
401
443
|
const PROCESS_LOG_ID = 'framework/process-log';
|
|
@@ -444,6 +486,106 @@ class DiscordAwarenessAccountingError extends Error {
|
|
|
444
486
|
}
|
|
445
487
|
}
|
|
446
488
|
|
|
489
|
+
/**
|
|
490
|
+
* Feasibility verdict for a runtime-settings patch, derived through the SAME
|
|
491
|
+
* settings→config mapping the live apply path uses (`Agent.planRuntimeSettings`
|
|
492
|
+
* → `previewContextSettings`). The `effective` verdict is for the budget the
|
|
493
|
+
* next compile actually plans at — for a paced descent that is the UNCHANGED
|
|
494
|
+
* live budget, with the descent target reported separately as `advisory`.
|
|
495
|
+
*/
|
|
496
|
+
export interface RuntimeSettingsPreview {
|
|
497
|
+
/** False when no verdict could be computed (no strategy support, preview
|
|
498
|
+
* overlap, stale branch generation, …) — see `reason`. Never silently
|
|
499
|
+
* swallowed: an unavailable preview is an answer, not an error. */
|
|
500
|
+
available: boolean;
|
|
501
|
+
reason?:
|
|
502
|
+
| 'no_preview_support'
|
|
503
|
+
| 'preview_in_flight'
|
|
504
|
+
| 'branch_generation_changed'
|
|
505
|
+
| 'no_adaptive_resolution'
|
|
506
|
+
| string;
|
|
507
|
+
path: 'immediate' | 'paced' | 'none';
|
|
508
|
+
/** The budget compiles currently plan at (baseline of the patch). */
|
|
509
|
+
liveBudgetTokens: number;
|
|
510
|
+
/** Verdict at the effective compile budget (spread of the strategy's
|
|
511
|
+
* PreviewResult: finalTokens, fits, headTokens, tailTokens, middleTokens,
|
|
512
|
+
* deepestLevel, exhausted, …). */
|
|
513
|
+
effective?: { budgetTokens: number; fits: boolean; finalTokens: number } & Record<string, unknown>;
|
|
514
|
+
/** Best-effort verdict at a paced-descent target. Advisory only — a target
|
|
515
|
+
* below the folded floor does not block the patch; the descent simply
|
|
516
|
+
* converges as far as the floor allows. */
|
|
517
|
+
advisory?: { targetTokens: number; fits: boolean } & Record<string, unknown>;
|
|
518
|
+
transition: 'stable' | 'converging' | 'blocked';
|
|
519
|
+
transitionReason?: string;
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
/**
|
|
523
|
+
* Thrown by `updateAgentRuntimeSettings` when an immediate budget change would
|
|
524
|
+
* put the agent in an un-compilable layout: the folded floor renders more
|
|
525
|
+
* tokens than the hard budget admits, so every subsequent compile would
|
|
526
|
+
* OverBudget-wedge the agent (2026-08-21: mythos, 550k→260k set live against a
|
|
527
|
+
* ~450k floor — hours of hard-down). Overridable with `{ allowInfeasible: true }`.
|
|
528
|
+
* Paced descents never throw this: a non-immediate decrease leaves the compile
|
|
529
|
+
* budget untouched and converges only as far as the floor allows.
|
|
530
|
+
*/
|
|
531
|
+
export class BudgetPreflightError extends Error {
|
|
532
|
+
constructor(
|
|
533
|
+
message: string,
|
|
534
|
+
readonly preview: RuntimeSettingsPreview,
|
|
535
|
+
) {
|
|
536
|
+
super(message);
|
|
537
|
+
this.name = 'BudgetPreflightError';
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
/** Operator-facing snapshot of the host's quiesce state (issue #122). */
|
|
542
|
+
export interface HostModeStatus {
|
|
543
|
+
quiesced: boolean;
|
|
544
|
+
reason?: string;
|
|
545
|
+
since?: number;
|
|
546
|
+
/** True when no turn is alive (activeTurnTokens empty — the token spans
|
|
547
|
+
* dequeue → settled teardown, strictly wider than activeStreams). */
|
|
548
|
+
drained: boolean;
|
|
549
|
+
activeTurns: number;
|
|
550
|
+
/** Wakes that own provider admission while waiting for an in-flight
|
|
551
|
+
* auxiliary call — no turn token yet, but a turn is one settle away. The
|
|
552
|
+
* admission continuation rechecks quiesce and requeues them; `drained`
|
|
553
|
+
* is false until it has. */
|
|
554
|
+
parkedAdmissions: number;
|
|
555
|
+
/** Inference requests parked by the quiesce wake gate (while quiesced), or
|
|
556
|
+
* simply queued (while serving — non-zero right after a resume until the
|
|
557
|
+
* scheduler's next pass consumes them). Continuations of a held turn
|
|
558
|
+
* (budget restart, tool results) are never counted. */
|
|
559
|
+
gatedRequests: number;
|
|
560
|
+
/** Background code-execution scripts still running. Quiesce does NOT stop
|
|
561
|
+
* them — they hold no turn token, so drain doesn't wait for them. Reported
|
|
562
|
+
* so the operator sees what is still acting during the window. */
|
|
563
|
+
backgroundScripts: number;
|
|
564
|
+
/** Context writes (module events, api message.send) withheld because the
|
|
565
|
+
* host is quiesced. Persisted while quiesced and flushed by resume(). */
|
|
566
|
+
deferredWrites: number;
|
|
567
|
+
/** Agents whose turn `abandon` could NOT cancel: the turn token is held but
|
|
568
|
+
* no stream exists yet (still in hooks/compile) or ever will (puppet tool
|
|
569
|
+
* turns). Present only after an abandon that left such turns behind. */
|
|
570
|
+
unabandonable?: string[];
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
/**
|
|
574
|
+
* Thrown by `resume()` when the current runtime settings do not compile for
|
|
575
|
+
* one or more agents — returning to service would OverBudget-wedge them on
|
|
576
|
+
* the first wake. Drain quarantine / advance merges to lower the floor, or
|
|
577
|
+
* pass `{ force: true }` after deciding the verdicts are acceptable.
|
|
578
|
+
*/
|
|
579
|
+
export class ResumeBlockedError extends Error {
|
|
580
|
+
constructor(
|
|
581
|
+
message: string,
|
|
582
|
+
readonly verdicts: Array<{ agentName: string; preview: RuntimeSettingsPreview }>,
|
|
583
|
+
) {
|
|
584
|
+
super(message);
|
|
585
|
+
this.name = 'ResumeBlockedError';
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
|
|
447
589
|
interface RedoEntry {
|
|
448
590
|
branchName: string;
|
|
449
591
|
checkpoint: TurnCheckpoint;
|
|
@@ -466,6 +608,85 @@ class DefaultInferencePolicy implements InferencePolicy {
|
|
|
466
608
|
}
|
|
467
609
|
}
|
|
468
610
|
|
|
611
|
+
/** Minimal read surface the live-surgery helpers need from a ContextManager. */
|
|
612
|
+
interface SurgeryReadableCm {
|
|
613
|
+
getMessageCount(): number;
|
|
614
|
+
getMessageWindow(
|
|
615
|
+
offset: number,
|
|
616
|
+
limit: number,
|
|
617
|
+
opts?: { resolveBlobs?: boolean; alignToBodyGroups?: boolean },
|
|
618
|
+
): { messages: Array<{ id: unknown; bodyGroupId?: string }>; startIndex: number };
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
const SURGERY_SCAN_WINDOW = 500;
|
|
622
|
+
|
|
623
|
+
/**
|
|
624
|
+
* Slot index of `messageId` on the active branch, or -1. Scans from the tail
|
|
625
|
+
* in blob-free windows: operator targets are almost always recent, and a
|
|
626
|
+
* full `getAllMessages()` re-inflates every attachment on the branch.
|
|
627
|
+
*/
|
|
628
|
+
function locateMessageIndex(cm: SurgeryReadableCm, messageId: string): number {
|
|
629
|
+
const hit = locateMessageIndices(cm, new Set([messageId])).get(messageId);
|
|
630
|
+
return hit ? hit.index : -1;
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
/** Tail-first scan resolving many ids at once; stops as soon as all are found. */
|
|
634
|
+
function locateMessageIndices(
|
|
635
|
+
cm: SurgeryReadableCm,
|
|
636
|
+
ids: Set<string>,
|
|
637
|
+
): Map<string, { index: number; bodyGroupId?: string }> {
|
|
638
|
+
const found = new Map<string, { index: number; bodyGroupId?: string }>();
|
|
639
|
+
const total = cm.getMessageCount();
|
|
640
|
+
for (let end = total; end > 0 && found.size < ids.size; end -= SURGERY_SCAN_WINDOW) {
|
|
641
|
+
const start = Math.max(0, end - SURGERY_SCAN_WINDOW);
|
|
642
|
+
const { messages, startIndex } = cm.getMessageWindow(start, end - start, { resolveBlobs: false });
|
|
643
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
644
|
+
const id = String(messages[i].id);
|
|
645
|
+
if (ids.has(id) && !found.has(id)) {
|
|
646
|
+
found.set(id, {
|
|
647
|
+
index: startIndex + i,
|
|
648
|
+
...(messages[i].bodyGroupId ? { bodyGroupId: messages[i].bodyGroupId } : {}),
|
|
649
|
+
});
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
return found;
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
/**
|
|
657
|
+
* Inclusive slot range of the body-group run containing `index` (the slot
|
|
658
|
+
* itself when it is not sharded). Delegates to the store's own
|
|
659
|
+
* `alignToBodyGroups` edge walk — one contiguity assumption, not two.
|
|
660
|
+
*/
|
|
661
|
+
function bodyGroupRun(
|
|
662
|
+
cm: SurgeryReadableCm,
|
|
663
|
+
index: number,
|
|
664
|
+
): { from: number; to: number; fromId: string; toId: string; group: boolean } {
|
|
665
|
+
const win = cm.getMessageWindow(index, 1, { alignToBodyGroups: true, resolveBlobs: false });
|
|
666
|
+
const from = win.startIndex;
|
|
667
|
+
const to = win.startIndex + win.messages.length - 1;
|
|
668
|
+
return {
|
|
669
|
+
from,
|
|
670
|
+
to,
|
|
671
|
+
fromId: String(win.messages[0].id),
|
|
672
|
+
toId: String(win.messages[win.messages.length - 1].id),
|
|
673
|
+
group: win.messages[0].bodyGroupId !== undefined,
|
|
674
|
+
};
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
function describeRequester(r: OperatorRequester | undefined): string {
|
|
678
|
+
if (!r) return 'unknown';
|
|
679
|
+
return `${r.name ?? r.id ?? 'unknown'} (${r.via})`;
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
function hostCommandRequester(serverId: string, params: HostCommandParams): OperatorRequester {
|
|
683
|
+
return {
|
|
684
|
+
via: `host-command:${serverId}`,
|
|
685
|
+
...(params.requesterName ? { name: params.requesterName } : {}),
|
|
686
|
+
...(params.requesterId ? { id: params.requesterId } : {}),
|
|
687
|
+
};
|
|
688
|
+
}
|
|
689
|
+
|
|
469
690
|
function isPermanentDiscordReactionFailure(message: string): boolean {
|
|
470
691
|
return /unknown message|unknown channel|missing access|missing permissions|missing permission|cannot access|channel .* not found|message .* not found/i
|
|
471
692
|
.test(message);
|
|
@@ -614,6 +835,14 @@ interface HostCommandParams {
|
|
|
614
835
|
maxRewinds?: number;
|
|
615
836
|
/** For the `unstick` command: raw channel id to post the outcome report to. */
|
|
616
837
|
channelId?: string;
|
|
838
|
+
/** For the `quiesce` command: operator-facing reason recorded in the mode. */
|
|
839
|
+
reason?: string;
|
|
840
|
+
/** For the `quiesce` command: drain window in ms (clamped to [1s, 10m]). */
|
|
841
|
+
timeoutMs?: number;
|
|
842
|
+
/** For the `quiesce` command: cancel undrained turns after the window. */
|
|
843
|
+
abandon?: boolean;
|
|
844
|
+
/** For the `resume` command: override a failing feasibility verdict. */
|
|
845
|
+
force?: boolean;
|
|
617
846
|
requesterId?: string;
|
|
618
847
|
requesterName?: string;
|
|
619
848
|
}
|
|
@@ -693,6 +922,17 @@ export class AgentFramework {
|
|
|
693
922
|
private activeTriggerChannels: Map<string, string> = new Map();
|
|
694
923
|
private running = false;
|
|
695
924
|
private loopPromise: Promise<void> | null = null;
|
|
925
|
+
/** Quiesce/maintenance mode (issue #122). DELIBERATELY separate from
|
|
926
|
+
* `running`, which is overloaded as the runLoop condition, the maintenance
|
|
927
|
+
* admission guard, AND the per-tick loop condition — expressing "paused" by
|
|
928
|
+
* clearing `running` would kill the very maintenance machinery quiesce
|
|
929
|
+
* exists to keep hot, plus store sync and tool-result processing. While
|
|
930
|
+
* quiesced: runLoop, syncTimer, maintenanceTimer, and the watchdog all keep
|
|
931
|
+
* running; only new turns (wake gate in processInferenceRequests) and MCPL
|
|
932
|
+
* data planes are held. */
|
|
933
|
+
private quiesced = false;
|
|
934
|
+
private quiesceReason?: string;
|
|
935
|
+
private quiescedAt?: number;
|
|
696
936
|
private traceListeners: TraceEventListener[] = [];
|
|
697
937
|
private syncIntervalMs: number;
|
|
698
938
|
private syncTimer: ReturnType<typeof setInterval> | null = null;
|
|
@@ -797,7 +1037,7 @@ export class AgentFramework {
|
|
|
797
1037
|
* `inference:exhausted` (which also pollutes the failure streak). Kept
|
|
798
1038
|
* separate from ephemeralRuns deliberately: endTurn/budget cancels happen
|
|
799
1039
|
* for resident agents too, and the key is per-stream, not per-agent. */
|
|
800
|
-
private frameworkCancelledStreams: Map<string, 'turn_ended' | 'budget_restart'> = new Map();
|
|
1040
|
+
private frameworkCancelledStreams: Map<string, 'turn_ended' | 'budget_restart' | 'quiesce_abandoned'> = new Map();
|
|
801
1041
|
/** Active runEphemeralToCompletion runs, keyed by agent name. */
|
|
802
1042
|
private ephemeralRuns: Map<string, EphemeralRun> = new Map();
|
|
803
1043
|
/** Ephemeral namespaces/names are single-generation for this framework
|
|
@@ -872,7 +1112,42 @@ export class AgentFramework {
|
|
|
872
1112
|
/** Tune-out coordinator (issue #77); non-null iff subconscious + channels. */
|
|
873
1113
|
private tuneOutCoordinator: TuneOutCoordinator | null = null;
|
|
874
1114
|
|
|
1115
|
+
/** Agents an abandon could not cancel (token held, no stream) — surfaced
|
|
1116
|
+
* in HostModeStatus until the next quiesce/resume. */
|
|
1117
|
+
private lastUnabandonable: string[] = [];
|
|
1118
|
+
/** True while the deferred-write queue has a persisted mirror (quiesced). */
|
|
1119
|
+
private deferredWritesPersisted = false;
|
|
1120
|
+
private deferredWritesCapWarned = false;
|
|
1121
|
+
/** Branch-independent recovery files (see FrameworkConfig.hostModePath /
|
|
1122
|
+
* deferredWritesPath). Undefined → branch-local slot fallback. */
|
|
1123
|
+
private hostModePath: string | undefined;
|
|
1124
|
+
private deferredWritesPath: string | undefined;
|
|
1125
|
+
private recoveryFallbackWarned = false;
|
|
1126
|
+
/**
|
|
1127
|
+
* Deferred writes that have been handed to a context manager but whose
|
|
1128
|
+
* chronicle state is NOT yet durably synced. They stay in the durable
|
|
1129
|
+
* recovery queue until `ackDeferredWrites()` syncs the store — chronicle
|
|
1130
|
+
* persists the slot-chain head only on sync(), so acknowledging on append
|
|
1131
|
+
* alone would let a hard exit forget an accepted message.
|
|
1132
|
+
*/
|
|
1133
|
+
private unackedDeferredWrites: DeferredWrite[] = [];
|
|
1134
|
+
/** Monotonic order stamp for deferred writes: the durable queue is always
|
|
1135
|
+
* written, restored, drained and flushed in `seq` order, whatever the
|
|
1136
|
+
* pending/un-acked split — a re-deferred entry keeps its place. */
|
|
1137
|
+
private deferredSeq = 0;
|
|
1138
|
+
/**
|
|
1139
|
+
* Per target agent: the store's message count when the OLDEST currently
|
|
1140
|
+
* un-acked entry for it was handed off. Persisted with the queue so a boot
|
|
1141
|
+
* after an interrupted flush knows exactly which slot range the batch
|
|
1142
|
+
* could occupy and dedups against all of it — never a fixed tail.
|
|
1143
|
+
*/
|
|
1144
|
+
private deferredScanFrom = new Map<string, number>();
|
|
875
1145
|
private deferredMessages: Array<{
|
|
1146
|
+
/** Durable identity for per-message flush acknowledgement: a flush
|
|
1147
|
+
* interrupted by a crash replays only the messages not yet acked. */
|
|
1148
|
+
id: string;
|
|
1149
|
+
/** Order stamp (see deferredSeq). */
|
|
1150
|
+
seq: number;
|
|
876
1151
|
participant: string;
|
|
877
1152
|
content: ContentBlock[];
|
|
878
1153
|
metadata?: MessageMetadata;
|
|
@@ -998,6 +1273,16 @@ export class AgentFramework {
|
|
|
998
1273
|
private discordAwarenessOutbox: DiscordAwarenessOutbox | null = null;
|
|
999
1274
|
private discordAwarenessEmoji = DEFAULT_DISCORD_AWARENESS_EMOJI;
|
|
1000
1275
|
private discordAwarenessDeadlineMs = DEFAULT_DISCORD_AWARENESS_DEADLINE_MS;
|
|
1276
|
+
/** Durable JSONL record of operator-initiated mutations (see operator-log.ts). */
|
|
1277
|
+
private readonly operatorLog: OperatorLog;
|
|
1278
|
+
/**
|
|
1279
|
+
* Store-wide admission barrier while a live surgery (rollback/suppress)
|
|
1280
|
+
* holds the store. Per-agent turn tokens cover the agents registered when
|
|
1281
|
+
* the reservation was taken; this flag covers everyone else — an ephemeral
|
|
1282
|
+
* agent admitted mid-switch, a wake for an agent created later — at the
|
|
1283
|
+
* scheduler, ephemeral admission and puppet entry points.
|
|
1284
|
+
*/
|
|
1285
|
+
private surgeryHold: { verb: string; agentName: string; since: number } | null = null;
|
|
1001
1286
|
/** Serialize per-server drains so reconnect and an online undo cannot race. */
|
|
1002
1287
|
private discordAwarenessDrains: Map<string, Promise<DiscordAwarenessDrainOutcome>> = new Map();
|
|
1003
1288
|
/** Framework-global inference gate; older generations cannot release it. */
|
|
@@ -1026,7 +1311,9 @@ export class AgentFramework {
|
|
|
1026
1311
|
discordAwarenessOutbox: DiscordAwarenessOutbox | null,
|
|
1027
1312
|
discordAwarenessEmoji: string,
|
|
1028
1313
|
discordAwarenessDeadlineMs: number,
|
|
1314
|
+
operatorLog: OperatorLog = new OperatorLog(undefined),
|
|
1029
1315
|
) {
|
|
1316
|
+
this.operatorLog = operatorLog;
|
|
1030
1317
|
this.store = store;
|
|
1031
1318
|
this.ownsStore = ownsStore;
|
|
1032
1319
|
this.membrane = membrane;
|
|
@@ -1088,6 +1375,11 @@ export class AgentFramework {
|
|
|
1088
1375
|
} catch {
|
|
1089
1376
|
// Already registered
|
|
1090
1377
|
}
|
|
1378
|
+
try {
|
|
1379
|
+
store.registerState({ id: DEFERRED_WRITES_ID, strategy: 'snapshot' });
|
|
1380
|
+
} catch {
|
|
1381
|
+
// Already registered
|
|
1382
|
+
}
|
|
1091
1383
|
|
|
1092
1384
|
try {
|
|
1093
1385
|
store.registerState({
|
|
@@ -1132,6 +1424,18 @@ export class AgentFramework {
|
|
|
1132
1424
|
? new DiscordAwarenessOutbox(discordAwarenessOutboxPath)
|
|
1133
1425
|
: null;
|
|
1134
1426
|
|
|
1427
|
+
const operatorLogPath = config.operatorLogPath === false
|
|
1428
|
+
? undefined
|
|
1429
|
+
: config.operatorLogPath
|
|
1430
|
+
?? (config.storePath ? defaultOperatorLogPath(config.storePath) : undefined);
|
|
1431
|
+
if (operatorLogPath === undefined && config.operatorLogPath !== false) {
|
|
1432
|
+
// An audit log that is silently off is worse than none: say so once.
|
|
1433
|
+
console.error(
|
|
1434
|
+
'[operator-log] disabled: no storePath and no operatorLogPath configured — ' +
|
|
1435
|
+
'operator actions will only be visible as operator:action traces',
|
|
1436
|
+
);
|
|
1437
|
+
}
|
|
1438
|
+
|
|
1135
1439
|
const framework = new AgentFramework(
|
|
1136
1440
|
store,
|
|
1137
1441
|
ownsStore,
|
|
@@ -1146,6 +1450,7 @@ export class AgentFramework {
|
|
|
1146
1450
|
discordAwarenessOutbox,
|
|
1147
1451
|
config.discordAwarenessEmoji ?? DEFAULT_DISCORD_AWARENESS_EMOJI,
|
|
1148
1452
|
normalizeDiscordAwarenessDeadline(config.discordAwarenessDeadlineMs),
|
|
1453
|
+
new OperatorLog(operatorLogPath),
|
|
1149
1454
|
);
|
|
1150
1455
|
|
|
1151
1456
|
// If an offline recovery process crashed after switching Chronicle but
|
|
@@ -1287,6 +1592,61 @@ export class AgentFramework {
|
|
|
1287
1592
|
framework.toolResultInlineMaxCharsOverride.set(agentName, cap);
|
|
1288
1593
|
}
|
|
1289
1594
|
|
|
1595
|
+
// Restore persisted quiesce mode (issue #122) BEFORE initializeMcpl: the
|
|
1596
|
+
// flag must be set before any data-plane barrier completion can run, or
|
|
1597
|
+
// the startup funnel would open the data planes on a host that shut down
|
|
1598
|
+
// mid-maintenance. Staged connections boot with both planes closed, so a
|
|
1599
|
+
// quiesced boot needs no re-pause — completeMcplDataPlaneGate consults
|
|
1600
|
+
// the flag and holds data planes (control planes come up normally). The
|
|
1601
|
+
// gate already exists at this point, so the suppression is wired here too.
|
|
1602
|
+
//
|
|
1603
|
+
// Both records live OUTSIDE branch history (recovery/ files next to the
|
|
1604
|
+
// store): a historical rollback must not be able to erase the marker of
|
|
1605
|
+
// the very surgery it belongs to, nor orphan the writes deferred by it.
|
|
1606
|
+
framework.hostModePath = config.hostModePath
|
|
1607
|
+
?? (config.storePath ? join(config.storePath, 'recovery', 'host-mode.json') : undefined);
|
|
1608
|
+
framework.deferredWritesPath = config.deferredWritesPath
|
|
1609
|
+
?? (config.storePath ? join(config.storePath, 'recovery', 'deferred-writes.json') : undefined);
|
|
1610
|
+
{
|
|
1611
|
+
const hostMode = framework.readHostMode();
|
|
1612
|
+
// Deferred writes are recovered REGARDLESS of the mode flag: a crash
|
|
1613
|
+
// between a resume's flag clear and the end of its flush must not
|
|
1614
|
+
// strand accepted messages. Per-message acks mean only the remainder
|
|
1615
|
+
// replays. Serving boot with a remainder → flush it right now (no turn
|
|
1616
|
+
// is alive yet, so this is a safe boundary).
|
|
1617
|
+
const restoredWrites = framework.restorePersistedDeferredWrites();
|
|
1618
|
+
if (!hostMode?.quiesced && restoredWrites > 0) {
|
|
1619
|
+
console.error(
|
|
1620
|
+
`[host-mode] ${restoredWrites} deferred context write(s) found at a serving boot ` +
|
|
1621
|
+
`(an earlier resume did not finish its flush) — landing them now`,
|
|
1622
|
+
);
|
|
1623
|
+
await framework.flushDeferredWrites('boot-recovery');
|
|
1624
|
+
}
|
|
1625
|
+
if (hostMode?.quiesced) {
|
|
1626
|
+
framework.quiesced = true;
|
|
1627
|
+
framework.quiesceReason = hostMode.reason;
|
|
1628
|
+
framework.quiescedAt = hostMode.since;
|
|
1629
|
+
framework.eventGate?.setQuiesced(true);
|
|
1630
|
+
console.error(
|
|
1631
|
+
`[host-mode] ============================================================\n` +
|
|
1632
|
+
`[host-mode] BOOTING QUIESCED (persisted${hostMode.reason ? `: ${hostMode.reason}` : ''}, ` +
|
|
1633
|
+
`since ${hostMode.since ? new Date(hostMode.since).toISOString() : 'unknown'}).\n` +
|
|
1634
|
+
`[host-mode] No wakes will start turns and MCPL data planes stay paused\n` +
|
|
1635
|
+
`[host-mode] until resume() — via host/command, the API server, or the\n` +
|
|
1636
|
+
`[host-mode] framework API. Wakes parked before the restart were in-memory\n` +
|
|
1637
|
+
`[host-mode] and are gone; recurring sources re-deliver on their own cadence.\n` +
|
|
1638
|
+
`[host-mode] ${restoredWrites} deferred context write(s) restored from the store;\n` +
|
|
1639
|
+
`[host-mode] they land at resume().\n` +
|
|
1640
|
+
`[host-mode] ============================================================`,
|
|
1641
|
+
);
|
|
1642
|
+
framework.emitTrace({
|
|
1643
|
+
type: 'host:quiesced_boot',
|
|
1644
|
+
...(hostMode.reason ? { reason: hostMode.reason } : {}),
|
|
1645
|
+
...(hostMode.since !== undefined ? { since: hostMode.since } : {}),
|
|
1646
|
+
});
|
|
1647
|
+
}
|
|
1648
|
+
}
|
|
1649
|
+
|
|
1290
1650
|
// Initialize MCPL subsystems if configured
|
|
1291
1651
|
if (config.mcplServers && config.mcplServers.length > 0) {
|
|
1292
1652
|
// Validate tool prefixes: no collisions with module names or between servers
|
|
@@ -1412,6 +1772,9 @@ export class AgentFramework {
|
|
|
1412
1772
|
*/
|
|
1413
1773
|
async stop(): Promise<void> {
|
|
1414
1774
|
this.running = false;
|
|
1775
|
+
// Flushed-but-unsynced deferred writes: sync and ack now, while the
|
|
1776
|
+
// store is still open, rather than leaving them to a reboot replay.
|
|
1777
|
+
this.ackDeferredWrites();
|
|
1415
1778
|
this.providerAdmissionClosed = true;
|
|
1416
1779
|
this.queue.close();
|
|
1417
1780
|
this.tuneOutCoordinator?.stop();
|
|
@@ -2166,21 +2529,74 @@ export class AgentFramework {
|
|
|
2166
2529
|
updateAgentRuntimeSettings(
|
|
2167
2530
|
agentName: string,
|
|
2168
2531
|
patch: AgentRuntimeSettingsPatch,
|
|
2169
|
-
opts?: { persist?: boolean },
|
|
2532
|
+
opts?: { persist?: boolean; allowInfeasible?: boolean; requester?: OperatorRequester; note?: string },
|
|
2170
2533
|
): AgentRuntimeSettingsSnapshot {
|
|
2171
2534
|
const agent = this.agents.get(agentName);
|
|
2172
2535
|
if (!agent) throw new Error(`Unknown agent: ${agentName}`);
|
|
2536
|
+
// Feasibility preflight — only a budget-bearing patch can wedge, and only
|
|
2537
|
+
// on the immediate path (a paced descent leaves the compile budget alone
|
|
2538
|
+
// and converges only as far as the folded floor allows). Boot restore
|
|
2539
|
+
// deliberately bypasses this wrapper (createAgent → restoreRuntimeSettings),
|
|
2540
|
+
// so a persisted-but-now-infeasible budget can never brick startup.
|
|
2541
|
+
if (patch.contextBudgetTokens !== undefined) {
|
|
2542
|
+
const preview = this.previewAgentRuntimeSettings(agentName, patch);
|
|
2543
|
+
if (preview.available && preview.effective) {
|
|
2544
|
+
const e = preview.effective;
|
|
2545
|
+
// Block only a LOWERING that doesn't fit. A no-op rewrite or an
|
|
2546
|
+
// increase on an already-over-floor agent never makes things worse —
|
|
2547
|
+
// refusing those would throw on exactly the wedged population the
|
|
2548
|
+
// guard exists to protect (and block stepwise remediation).
|
|
2549
|
+
const lowering = patch.contextBudgetTokens! < preview.liveBudgetTokens;
|
|
2550
|
+
if (preview.path === 'immediate' && !e.fits && lowering) {
|
|
2551
|
+
const msg =
|
|
2552
|
+
`[budget-preflight] contextBudgetTokens=${patch.contextBudgetTokens} would NOT fit ` +
|
|
2553
|
+
`${agentName}: the folded floor renders ${e.finalTokens} tokens against hard budget ` +
|
|
2554
|
+
`${e.budgetTokens} (head=${e.headTokens ?? '?'} tail=${e.tailTokens ?? '?'} ` +
|
|
2555
|
+
`middle=${e.middleTokens ?? '?'}, deepest L${e.deepestLevel ?? '?'}` +
|
|
2556
|
+
`${e.exhausted ? ', picker exhausted' : ''}). Every compile at this budget would ` +
|
|
2557
|
+
`OverBudget-wedge the agent. Lower the floor first (drain compression quarantine / ` +
|
|
2558
|
+
`advance the merge ladder), use a paced descent (no \`immediate\`), or pass allowInfeasible.`;
|
|
2559
|
+
if (!opts?.allowInfeasible) throw new BudgetPreflightError(msg, preview);
|
|
2560
|
+
console.warn(`${msg} — applying anyway (allowInfeasible).`);
|
|
2561
|
+
} else if (preview.path === 'immediate' && !e.fits) {
|
|
2562
|
+
console.warn(
|
|
2563
|
+
`[budget-preflight] ${agentName} remains over the folded floor at ` +
|
|
2564
|
+
`${patch.contextBudgetTokens} (floor renders ${e.finalTokens}); this patch does ` +
|
|
2565
|
+
`not lower the budget, so applying — drain quarantine / advance merges to clear it.`,
|
|
2566
|
+
);
|
|
2567
|
+
} else if (preview.path === 'paced' && preview.advisory && !preview.advisory.fits) {
|
|
2568
|
+
console.warn(
|
|
2569
|
+
`[budget-preflight] paced descent for ${agentName} targets ` +
|
|
2570
|
+
`${preview.advisory.targetTokens} tokens, below the current folded floor — the ` +
|
|
2571
|
+
`transition will converge only as far as the floor allows (drain quarantine / ` +
|
|
2572
|
+
`advance merges to go lower). Applying; the live compile budget is unchanged.`,
|
|
2573
|
+
);
|
|
2574
|
+
}
|
|
2575
|
+
} else if (preview.reason) {
|
|
2576
|
+
console.warn(
|
|
2577
|
+
`[budget-preflight] preview unavailable for ${agentName} (${preview.reason}); ` +
|
|
2578
|
+
`applying without preflight`,
|
|
2579
|
+
);
|
|
2580
|
+
}
|
|
2581
|
+
}
|
|
2173
2582
|
const result = agent.updateRuntimeSettings(patch);
|
|
2174
2583
|
if (opts?.persist !== false) {
|
|
2175
2584
|
this.persistAgentRuntimeSettings(agentName, agent.getRuntimeSettingsOverrides());
|
|
2176
2585
|
}
|
|
2586
|
+
this.recordOperatorAction({
|
|
2587
|
+
kind: 'settings-update',
|
|
2588
|
+
agent: agentName,
|
|
2589
|
+
...(opts?.requester ? { requester: opts.requester } : {}),
|
|
2590
|
+
...(opts?.note ? { note: opts.note } : {}),
|
|
2591
|
+
params: { patch: { ...patch }, persist: opts?.persist !== false },
|
|
2592
|
+
});
|
|
2177
2593
|
return result;
|
|
2178
2594
|
}
|
|
2179
2595
|
|
|
2180
2596
|
resetAgentRuntimeSettings(
|
|
2181
2597
|
agentName: string,
|
|
2182
2598
|
keys?: Array<keyof AgentRuntimeSettingsPatch>,
|
|
2183
|
-
opts?: { persist?: boolean },
|
|
2599
|
+
opts?: { persist?: boolean; requester?: OperatorRequester; note?: string },
|
|
2184
2600
|
): AgentRuntimeSettingsSnapshot {
|
|
2185
2601
|
const agent = this.agents.get(agentName);
|
|
2186
2602
|
if (!agent) throw new Error(`Unknown agent: ${agentName}`);
|
|
@@ -2188,17 +2604,658 @@ export class AgentFramework {
|
|
|
2188
2604
|
if (opts?.persist !== false) {
|
|
2189
2605
|
this.persistAgentRuntimeSettings(agentName, agent.getRuntimeSettingsOverrides());
|
|
2190
2606
|
}
|
|
2607
|
+
this.recordOperatorAction({
|
|
2608
|
+
kind: 'settings-reset',
|
|
2609
|
+
agent: agentName,
|
|
2610
|
+
...(opts?.requester ? { requester: opts.requester } : {}),
|
|
2611
|
+
...(opts?.note ? { note: opts.note } : {}),
|
|
2612
|
+
params: { keys: keys ?? 'all', persist: opts?.persist !== false },
|
|
2613
|
+
});
|
|
2191
2614
|
return result;
|
|
2192
2615
|
}
|
|
2193
2616
|
|
|
2194
|
-
cancelAgentRuntimeSettingsTransition(
|
|
2617
|
+
cancelAgentRuntimeSettingsTransition(
|
|
2618
|
+
agentName: string,
|
|
2619
|
+
opts?: { requester?: OperatorRequester },
|
|
2620
|
+
): AgentRuntimeSettingsSnapshot {
|
|
2195
2621
|
const agent = this.agents.get(agentName);
|
|
2196
2622
|
if (!agent) throw new Error(`Unknown agent: ${agentName}`);
|
|
2197
2623
|
const result = agent.cancelRuntimeSettingsTransition();
|
|
2198
2624
|
this.persistAgentRuntimeSettings(agentName, agent.getRuntimeSettingsOverrides());
|
|
2625
|
+
this.recordOperatorAction({
|
|
2626
|
+
kind: 'settings-cancel-transition',
|
|
2627
|
+
agent: agentName,
|
|
2628
|
+
...(opts?.requester ? { requester: opts.requester } : {}),
|
|
2629
|
+
});
|
|
2199
2630
|
return result;
|
|
2200
2631
|
}
|
|
2201
2632
|
|
|
2633
|
+
// -------------------------------------------------------------------------
|
|
2634
|
+
// Host quiesce / maintenance mode (issue #122)
|
|
2635
|
+
//
|
|
2636
|
+
// Pause the inference thread and MCPL data planes while keeping the
|
|
2637
|
+
// framework + context managers + membrane loaded and hot, so maintenance
|
|
2638
|
+
// (compression ticks, refold, quarantine drains, budget descents) runs
|
|
2639
|
+
// through the exact machinery the agent uses live — with live config, live
|
|
2640
|
+
// tool definitions, and full llm-calls logging — instead of offline rigs
|
|
2641
|
+
// that re-derive all of it and drift.
|
|
2642
|
+
//
|
|
2643
|
+
// While quiesced: runLoop, store sync, the maintenance timer, and the
|
|
2644
|
+
// liveness watchdog all keep running. Wakes park in pendingRequests
|
|
2645
|
+
// (coalesced per reason) and fire at resume; MCPL events buffer on the
|
|
2646
|
+
// paused data planes; gate debounces buffer in the gate. Background
|
|
2647
|
+
// code-execution scripts are NOT stopped (they hold no turn token) — they
|
|
2648
|
+
// are reported in the status so the operator sees what still acts.
|
|
2649
|
+
//
|
|
2650
|
+
// NOTE: do not call quiesce() from inside a queue event handler — the drain
|
|
2651
|
+
// wait depends on the event loop continuing to run.
|
|
2652
|
+
// -------------------------------------------------------------------------
|
|
2653
|
+
|
|
2654
|
+
/**
|
|
2655
|
+
* Wakes that passed the scheduler and now own provider admission while
|
|
2656
|
+
* waiting for an in-flight auxiliary call to settle — no turn token yet,
|
|
2657
|
+
* so `activeTurnTokens` alone would call the host drained while a turn is
|
|
2658
|
+
* one promise-resolution away from starting. The admission continuation
|
|
2659
|
+
* rechecks quiesce and requeues, but until it runs the wake is live.
|
|
2660
|
+
*/
|
|
2661
|
+
private parkedAdmissionCount(): number {
|
|
2662
|
+
let n = 0;
|
|
2663
|
+
for (const [agentName, gate] of this.providerGates) {
|
|
2664
|
+
if (gate.primaryDepth > 0 && !this.activeTurnTokens.has(agentName)) n++;
|
|
2665
|
+
}
|
|
2666
|
+
return n;
|
|
2667
|
+
}
|
|
2668
|
+
|
|
2669
|
+
getHostModeStatus(): HostModeStatus {
|
|
2670
|
+
const parkedAdmissions = this.parkedAdmissionCount();
|
|
2671
|
+
return {
|
|
2672
|
+
quiesced: this.quiesced,
|
|
2673
|
+
...(this.quiesceReason ? { reason: this.quiesceReason } : {}),
|
|
2674
|
+
...(this.quiescedAt !== undefined ? { since: this.quiescedAt } : {}),
|
|
2675
|
+
drained: this.activeTurnTokens.size === 0 && parkedAdmissions === 0,
|
|
2676
|
+
activeTurns: this.activeTurnTokens.size,
|
|
2677
|
+
parkedAdmissions,
|
|
2678
|
+
gatedRequests: this.pendingRequests.filter((r) => !isTurnContinuation(r.reason)).length,
|
|
2679
|
+
backgroundScripts: [...this.backgroundScripts.values()]
|
|
2680
|
+
.filter((record) => record.status === 'running').length,
|
|
2681
|
+
deferredWrites: this.deferredMessages.length,
|
|
2682
|
+
...(this.lastUnabandonable.length > 0 ? { unabandonable: [...this.lastUnabandonable] } : {}),
|
|
2683
|
+
};
|
|
2684
|
+
}
|
|
2685
|
+
|
|
2686
|
+
/** Drain window bounds shared by every ingress (framework API, WS, HTTP,
|
|
2687
|
+
* host/command). Below the floor a drain is meaningless; above the ceiling
|
|
2688
|
+
* an awaited call would poll for days. */
|
|
2689
|
+
static readonly QUIESCE_TIMEOUT_MIN_MS = 1_000;
|
|
2690
|
+
static readonly QUIESCE_TIMEOUT_MAX_MS = 600_000;
|
|
2691
|
+
static readonly QUIESCE_TIMEOUT_DEFAULT_MS = 120_000;
|
|
2692
|
+
|
|
2693
|
+
/**
|
|
2694
|
+
* Enter quiesce mode: persist the flag, suppress gate deliveries, pause all
|
|
2695
|
+
* MCPL data planes, and wait for in-flight turns to settle (turn-alive is
|
|
2696
|
+
* `activeTurnTokens`, which spans dequeue → settled teardown). Idempotent.
|
|
2697
|
+
*
|
|
2698
|
+
* On drain timeout the host STAYS quiesced (`drained: false` in the
|
|
2699
|
+
* result); with `abandon: true` the undrained streams are cancelled via a
|
|
2700
|
+
* dedicated cancel kind that settles the turn without feeding the
|
|
2701
|
+
* inference-failure accounting.
|
|
2702
|
+
*/
|
|
2703
|
+
async quiesce(opts?: {
|
|
2704
|
+
reason?: string;
|
|
2705
|
+
timeoutMs?: number;
|
|
2706
|
+
abandon?: boolean;
|
|
2707
|
+
}): Promise<HostModeStatus> {
|
|
2708
|
+
if (this.quiesced) {
|
|
2709
|
+
// Idempotent — EXCEPT abandon escalation: a first quiesce that timed
|
|
2710
|
+
// out undrained must be escalatable with a second quiesce({abandon})
|
|
2711
|
+
// without resume()+re-quiesce (which would reopen data planes and
|
|
2712
|
+
// release parked wakes mid-surgery).
|
|
2713
|
+
if (opts?.abandon && this.activeTurnTokens.size > 0) {
|
|
2714
|
+
const unabandonable = await this.abandonActiveTurns();
|
|
2715
|
+
this.emitTrace({
|
|
2716
|
+
type: 'host:quiesce',
|
|
2717
|
+
...(this.quiesceReason ? { reason: this.quiesceReason } : {}),
|
|
2718
|
+
drained: this.activeTurnTokens.size === 0,
|
|
2719
|
+
activeTurns: this.activeTurnTokens.size,
|
|
2720
|
+
abandoned: true,
|
|
2721
|
+
...(unabandonable.length > 0 ? { unabandonable } : {}),
|
|
2722
|
+
});
|
|
2723
|
+
}
|
|
2724
|
+
return this.getHostModeStatus();
|
|
2725
|
+
}
|
|
2726
|
+
this.quiesced = true;
|
|
2727
|
+
this.quiesceReason = opts?.reason;
|
|
2728
|
+
this.quiescedAt = Date.now();
|
|
2729
|
+
this.lastUnabandonable = [];
|
|
2730
|
+
this.persistHostMode({
|
|
2731
|
+
quiesced: true,
|
|
2732
|
+
...(opts?.reason ? { reason: opts.reason } : {}),
|
|
2733
|
+
since: this.quiescedAt,
|
|
2734
|
+
});
|
|
2735
|
+
console.error(
|
|
2736
|
+
`[host-mode] quiescing${opts?.reason ? ` (${opts.reason})` : ''}: parking wakes, ` +
|
|
2737
|
+
`pausing MCPL data planes, draining ${this.activeTurnTokens.size} in-flight turn(s)`,
|
|
2738
|
+
);
|
|
2739
|
+
this.eventGate?.setQuiesced(true);
|
|
2740
|
+
for (const connection of this.mcplServerRegistry?.getAllServers() ?? []) {
|
|
2741
|
+
connection.pauseDataPlane();
|
|
2742
|
+
}
|
|
2743
|
+
|
|
2744
|
+
// Clamp HERE, for every ingress. NaN-proof too: timeoutMs arrives via
|
|
2745
|
+
// Number()/blind casts, and Math.max(1_000, NaN) is NaN — which would
|
|
2746
|
+
// silently collapse the drain window to zero (`Date.now() < NaN` is
|
|
2747
|
+
// false). The ceiling matters because this call is awaited: an
|
|
2748
|
+
// unbounded value polls the event loop for as long as it says.
|
|
2749
|
+
const rawTimeout = opts?.timeoutMs;
|
|
2750
|
+
const timeoutMs = typeof rawTimeout === 'number' && Number.isFinite(rawTimeout)
|
|
2751
|
+
? Math.max(
|
|
2752
|
+
AgentFramework.QUIESCE_TIMEOUT_MIN_MS,
|
|
2753
|
+
Math.min(AgentFramework.QUIESCE_TIMEOUT_MAX_MS, Math.floor(rawTimeout)),
|
|
2754
|
+
)
|
|
2755
|
+
: AgentFramework.QUIESCE_TIMEOUT_DEFAULT_MS;
|
|
2756
|
+
const deadline = Date.now() + timeoutMs;
|
|
2757
|
+
// Drain = no turn token AND no wake parked on provider admission (the
|
|
2758
|
+
// latter becomes a turn the moment its auxiliary settles unless the
|
|
2759
|
+
// continuation's quiesce recheck requeues it — wait for that to happen).
|
|
2760
|
+
while (
|
|
2761
|
+
this.quiesced &&
|
|
2762
|
+
(this.activeTurnTokens.size > 0 || this.parkedAdmissionCount() > 0) &&
|
|
2763
|
+
Date.now() < deadline
|
|
2764
|
+
) {
|
|
2765
|
+
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
2766
|
+
}
|
|
2767
|
+
|
|
2768
|
+
// A concurrent resume() ended this window while we waited. Nothing here
|
|
2769
|
+
// may act any more — an abandon now would cancel a turn that started
|
|
2770
|
+
// legitimately AFTER the resume, and the trace/log would describe a mode
|
|
2771
|
+
// the host is no longer in.
|
|
2772
|
+
if (!this.quiesced) {
|
|
2773
|
+
console.error('[host-mode] quiesce superseded by a concurrent resume during drain');
|
|
2774
|
+
return this.getHostModeStatus();
|
|
2775
|
+
}
|
|
2776
|
+
|
|
2777
|
+
let abandoned = false;
|
|
2778
|
+
let unabandonable: string[] = [];
|
|
2779
|
+
if (this.activeTurnTokens.size > 0 && opts?.abandon) {
|
|
2780
|
+
abandoned = true;
|
|
2781
|
+
unabandonable = await this.abandonActiveTurns();
|
|
2782
|
+
if (!this.quiesced) return this.getHostModeStatus();
|
|
2783
|
+
}
|
|
2784
|
+
|
|
2785
|
+
const status = this.getHostModeStatus();
|
|
2786
|
+
this.emitTrace({
|
|
2787
|
+
type: 'host:quiesce',
|
|
2788
|
+
...(this.quiesceReason ? { reason: this.quiesceReason } : {}),
|
|
2789
|
+
drained: status.drained,
|
|
2790
|
+
activeTurns: status.activeTurns,
|
|
2791
|
+
...(status.parkedAdmissions > 0 ? { parkedAdmissions: status.parkedAdmissions } : {}),
|
|
2792
|
+
...(abandoned ? { abandoned: true } : {}),
|
|
2793
|
+
...(unabandonable.length > 0 ? { unabandonable } : {}),
|
|
2794
|
+
});
|
|
2795
|
+
console.error(
|
|
2796
|
+
status.drained
|
|
2797
|
+
? `[host-mode] quiesced — drained, ${status.gatedRequests} wake(s) parked`
|
|
2798
|
+
: `[host-mode] quiesced but drain ${abandoned ? 'needed abandon and' : 'timed out —'} ` +
|
|
2799
|
+
`${status.activeTurns} turn(s) still alive` +
|
|
2800
|
+
(status.parkedAdmissions > 0
|
|
2801
|
+
? `, ${status.parkedAdmissions} wake(s) parked on provider admission`
|
|
2802
|
+
: ''),
|
|
2803
|
+
);
|
|
2804
|
+
return status;
|
|
2805
|
+
}
|
|
2806
|
+
|
|
2807
|
+
/** Cancel every turn-alive stream via the quiesce_abandoned kind and give
|
|
2808
|
+
* teardown a bounded grace to settle. Shared by the drain-timeout path and
|
|
2809
|
+
* the already-quiesced escalation path. */
|
|
2810
|
+
private async abandonActiveTurns(): Promise<string[]> {
|
|
2811
|
+
const unabandonable: string[] = [];
|
|
2812
|
+
for (const agentName of [...this.activeTurnTokens.keys()]) {
|
|
2813
|
+
const agent = this.agents.get(agentName);
|
|
2814
|
+
const state = agent?.state;
|
|
2815
|
+
const stream = state && 'stream' in state ? state.stream : undefined;
|
|
2816
|
+
if (agent && stream) {
|
|
2817
|
+
console.error(`[host-mode] abandoning in-flight turn for ${agentName}`);
|
|
2818
|
+
this.frameworkCancelledStreams.set(
|
|
2819
|
+
`${agent.name}:${agent.streamId}`,
|
|
2820
|
+
'quiesce_abandoned',
|
|
2821
|
+
);
|
|
2822
|
+
stream.cancel();
|
|
2823
|
+
} else {
|
|
2824
|
+
// Token held, no stream to cancel: the turn is between dequeue and
|
|
2825
|
+
// stream registration (hooks/compile), or it is a puppet tool turn,
|
|
2826
|
+
// which never has one. Say so — silence here reads as "abandon acted
|
|
2827
|
+
// and teardown is slow", which is a different operator decision.
|
|
2828
|
+
unabandonable.push(agentName);
|
|
2829
|
+
console.error(
|
|
2830
|
+
`[host-mode] cannot abandon turn for ${agentName}: turn token held with no stream ` +
|
|
2831
|
+
`(status=${state?.status ?? 'unknown'}) — it must settle on its own`,
|
|
2832
|
+
);
|
|
2833
|
+
}
|
|
2834
|
+
}
|
|
2835
|
+
// Bounded grace for the cancelled streams' teardown to settle.
|
|
2836
|
+
const grace = Date.now() + 10_000;
|
|
2837
|
+
while (this.quiesced && this.activeTurnTokens.size > 0 && Date.now() < grace) {
|
|
2838
|
+
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
2839
|
+
}
|
|
2840
|
+
// Only turns still alive after the grace are worth reporting.
|
|
2841
|
+
this.lastUnabandonable = unabandonable.filter((name) => this.activeTurnTokens.has(name));
|
|
2842
|
+
return this.lastUnabandonable;
|
|
2843
|
+
}
|
|
2844
|
+
|
|
2845
|
+
/**
|
|
2846
|
+
* Leave quiesce mode. Gates on a FRESH feasibility preview of every agent's
|
|
2847
|
+
* CURRENT settings — maintenance may have moved the folded floor and the
|
|
2848
|
+
* branch generation, so verdicts are computed at resume time, never reused.
|
|
2849
|
+
* Throws ResumeBlockedError (with all failing verdicts) unless `force`.
|
|
2850
|
+
* Unavailable previews warn and pass — refusing to resume because the
|
|
2851
|
+
* strategy cannot preview would hold hosts hostage to a diagnostic.
|
|
2852
|
+
*/
|
|
2853
|
+
async resume(opts?: { force?: boolean }): Promise<HostModeStatus> {
|
|
2854
|
+
if (!this.quiesced) return this.getHostModeStatus();
|
|
2855
|
+
|
|
2856
|
+
// The canonical operator flow is `maintain; resume`, and a timer-driven
|
|
2857
|
+
// pass may still be mid-flight: previewContext refuses to overlap one
|
|
2858
|
+
// ("already running"), which used to degrade the gate to "no opinion".
|
|
2859
|
+
// Wait for the pass first, then retry a few times for the case where a
|
|
2860
|
+
// fresh pass started in between.
|
|
2861
|
+
if (this.maintenancePass) {
|
|
2862
|
+
await this.maintenancePass.catch(() => {});
|
|
2863
|
+
}
|
|
2864
|
+
const failing: Array<{ agentName: string; preview: RuntimeSettingsPreview }> = [];
|
|
2865
|
+
for (const agentName of this.agents.keys()) {
|
|
2866
|
+
let preview = this.previewAgentRuntimeSettings(agentName);
|
|
2867
|
+
for (let attempt = 0; attempt < 8 && !preview.available && preview.reason === 'preview_in_flight'; attempt++) {
|
|
2868
|
+
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
2869
|
+
if (this.maintenancePass) await this.maintenancePass.catch(() => {});
|
|
2870
|
+
preview = this.previewAgentRuntimeSettings(agentName);
|
|
2871
|
+
}
|
|
2872
|
+
if (preview.available && preview.effective && !preview.effective.fits) {
|
|
2873
|
+
failing.push({ agentName, preview });
|
|
2874
|
+
} else if (!preview.available && preview.reason !== 'no_preview_support') {
|
|
2875
|
+
console.warn(
|
|
2876
|
+
`[host-mode] resume: feasibility preview unavailable for ${agentName} ` +
|
|
2877
|
+
`(${preview.reason}) — proceeding without a verdict`,
|
|
2878
|
+
);
|
|
2879
|
+
}
|
|
2880
|
+
}
|
|
2881
|
+
if (failing.length > 0 && !opts?.force) {
|
|
2882
|
+
const detail = failing.map(({ agentName, preview }) =>
|
|
2883
|
+
`${agentName}: folded floor ${preview.effective!.finalTokens} > hard budget ` +
|
|
2884
|
+
`${preview.effective!.budgetTokens}` +
|
|
2885
|
+
(preview.transition === 'blocked'
|
|
2886
|
+
? ` (transition blocked: ${preview.transitionReason ?? 'unknown'})`
|
|
2887
|
+
: ''),
|
|
2888
|
+
).join('; ');
|
|
2889
|
+
throw new ResumeBlockedError(
|
|
2890
|
+
`resume refused: returning to service would OverBudget-wedge — ${detail}. ` +
|
|
2891
|
+
`Drain compression quarantine / advance the merge ladder to lower the floor ` +
|
|
2892
|
+
`(maintenanceTick()), or resume({ force: true }).`,
|
|
2893
|
+
failing,
|
|
2894
|
+
);
|
|
2895
|
+
}
|
|
2896
|
+
if (failing.length > 0) {
|
|
2897
|
+
console.warn(
|
|
2898
|
+
`[host-mode] resume FORCED past ${failing.length} failing feasibility verdict(s)`,
|
|
2899
|
+
);
|
|
2900
|
+
}
|
|
2901
|
+
|
|
2902
|
+
const releasedRequests = this.pendingRequests.length;
|
|
2903
|
+
// In-memory first: addMessage must stop deferring so the flush below can
|
|
2904
|
+
// land. The DURABLE flag is cleared only after the flush completes — a
|
|
2905
|
+
// crash in between then boots quiesced with the un-acked remainder
|
|
2906
|
+
// restored (and a serving boot recovers any remainder anyway).
|
|
2907
|
+
this.quiesced = false;
|
|
2908
|
+
this.quiesceReason = undefined;
|
|
2909
|
+
this.quiescedAt = undefined;
|
|
2910
|
+
this.lastUnabandonable = [];
|
|
2911
|
+
this.eventGate?.setQuiesced(false);
|
|
2912
|
+
|
|
2913
|
+
try {
|
|
2914
|
+
// Flush context writes deferred by the quiesce window (module events,
|
|
2915
|
+
// api message.send) — with no turn alive there is no other flush
|
|
2916
|
+
// point. Per-message try/catch and per-message durable ack (see
|
|
2917
|
+
// flushDeferredWrites); nothing here may reach the plane reopen below.
|
|
2918
|
+
await this.flushDeferredWrites('resume');
|
|
2919
|
+
this.persistHostMode(null);
|
|
2920
|
+
} finally {
|
|
2921
|
+
// Reopen MCPL data planes through the existing barrier funnel — NOT a
|
|
2922
|
+
// bespoke ready() loop. The funnel inherits completeMcplDataPlaneGate's
|
|
2923
|
+
// nested-install guard (a flushed tools-list-changed can install a
|
|
2924
|
+
// newer barrier mid-flush), drains any awareness work accumulated
|
|
2925
|
+
// during the window before opening, and replaces a stale failed
|
|
2926
|
+
// barrier by identity. In `finally` so that nothing above can leave
|
|
2927
|
+
// the host un-quiesced with every data plane still paused.
|
|
2928
|
+
if (this.mcplServerRegistry) {
|
|
2929
|
+
const barrier = this.installMcplDataPlaneGate();
|
|
2930
|
+
this.releaseMcplDataPlaneGate(barrier);
|
|
2931
|
+
try {
|
|
2932
|
+
await barrier.promise;
|
|
2933
|
+
this.completeMcplDataPlaneGate(barrier);
|
|
2934
|
+
} catch (error) {
|
|
2935
|
+
// The host IS resumed — don't rethrow. failMcplDataPlaneGate
|
|
2936
|
+
// recycles the connections, and their reconnect flows re-run the
|
|
2937
|
+
// funnel with quiesced=false, self-healing the data planes.
|
|
2938
|
+
await this.failMcplDataPlaneGate(barrier, 'quiesce resume', error);
|
|
2939
|
+
}
|
|
2940
|
+
}
|
|
2941
|
+
|
|
2942
|
+
this.emitTrace({
|
|
2943
|
+
type: 'host:resume',
|
|
2944
|
+
...(opts?.force && failing.length > 0 ? { forced: true } : {}),
|
|
2945
|
+
releasedRequests,
|
|
2946
|
+
});
|
|
2947
|
+
// NOTE: parked wakes and gate buffers are in-memory only — a restart
|
|
2948
|
+
// mid-quiesce boots quiesced but with these queues empty (sources like
|
|
2949
|
+
// heartbeats re-deliver on their own cadence; one-shot wakes are lost).
|
|
2950
|
+
// Deferred context WRITES are persisted while quiesced and survive.
|
|
2951
|
+
console.error(`[host-mode] resumed — ${releasedRequests} parked wake(s) released`);
|
|
2952
|
+
}
|
|
2953
|
+
return this.getHostModeStatus();
|
|
2954
|
+
}
|
|
2955
|
+
|
|
2956
|
+
/**
|
|
2957
|
+
* Land every deferred write whose target has no turn alive, one at a time,
|
|
2958
|
+
* acknowledging each durably as it lands (the persisted queue is rewritten
|
|
2959
|
+
* after every message). A crash mid-flush therefore replays exactly the
|
|
2960
|
+
* un-acked remainder at the next boot — no loss, no duplicates. A write
|
|
2961
|
+
* the store REJECTS is logged and acked too: replaying a poison message
|
|
2962
|
+
* forever would wedge every later boot on it.
|
|
2963
|
+
*
|
|
2964
|
+
* Per-target: an agent whose turn is still alive keeps ITS messages
|
|
2965
|
+
* deferred for its own turn boundary; everyone else's flush now.
|
|
2966
|
+
*/
|
|
2967
|
+
private async flushDeferredWrites(label: string): Promise<void> {
|
|
2968
|
+
if (this.deferredMessages.length === 0) {
|
|
2969
|
+
this.persistDeferredWrites();
|
|
2970
|
+
return;
|
|
2971
|
+
}
|
|
2972
|
+
const keep: typeof this.deferredMessages = [];
|
|
2973
|
+
const flush: typeof this.deferredMessages = [];
|
|
2974
|
+
for (const msg of this.deferredMessages) {
|
|
2975
|
+
const target = msg.forAgent ?? this.primaryAgentName;
|
|
2976
|
+
(target && this.activeTurnTokens.has(target) ? keep : flush).push(msg);
|
|
2977
|
+
}
|
|
2978
|
+
// Hand-off: the batch leaves the pending queue and enters the un-acked
|
|
2979
|
+
// set. The DURABLE queue (pending + un-acked) is unchanged by this, so a
|
|
2980
|
+
// crash anywhere below replays the whole batch, and the deferredWriteId
|
|
2981
|
+
// stamped on each stored message lets boot skip the ones that did land.
|
|
2982
|
+
flush.sort(bySeq);
|
|
2983
|
+
this.deferredMessages = keep;
|
|
2984
|
+
this.handOffDeferredWrites(flush);
|
|
2985
|
+
let stored = 0;
|
|
2986
|
+
for (const msg of flush) {
|
|
2987
|
+
try {
|
|
2988
|
+
this.addMessage(msg.participant, msg.content, msg.metadata, {
|
|
2989
|
+
deferredWriteId: msg.id,
|
|
2990
|
+
...(msg.forAgent ? { forAgent: msg.forAgent } : {}),
|
|
2991
|
+
});
|
|
2992
|
+
stored++;
|
|
2993
|
+
} catch (err) {
|
|
2994
|
+
console.error(
|
|
2995
|
+
`[host-mode] ${label}: failed to store a deferred context write ` +
|
|
2996
|
+
`(participant=${msg.participant}, forAgent=${msg.forAgent ?? 'primary'}):`,
|
|
2997
|
+
err,
|
|
2998
|
+
);
|
|
2999
|
+
}
|
|
3000
|
+
}
|
|
3001
|
+
// Ack = sync the chronicle FIRST, then rewrite the durable queue without
|
|
3002
|
+
// the batch. Never the other way round.
|
|
3003
|
+
this.ackDeferredWrites();
|
|
3004
|
+
console.error(
|
|
3005
|
+
`[host-mode] ${label}: flushed ${stored}/${flush.length} deferred context write(s)` +
|
|
3006
|
+
(keep.length > 0 ? `, ${keep.length} kept for a still-alive turn` : ''),
|
|
3007
|
+
);
|
|
3008
|
+
}
|
|
3009
|
+
|
|
3010
|
+
/**
|
|
3011
|
+
* Acknowledge every deferred write that has been handed to a context
|
|
3012
|
+
* manager: sync the chronicle so the appended slots are durable, and only
|
|
3013
|
+
* then drop them from the recovery queue. If the sync fails they stay in
|
|
3014
|
+
* the durable queue (the next ack, or stop(), retries); a reboot in that
|
|
3015
|
+
* state replays them, deduplicated by `deferredWriteId` for any that did
|
|
3016
|
+
* reach disk. Idempotent and cheap when nothing is un-acked.
|
|
3017
|
+
*/
|
|
3018
|
+
private ackDeferredWrites(): void {
|
|
3019
|
+
if (this.unackedDeferredWrites.length === 0) return;
|
|
3020
|
+
// Nothing durable to reconcile against unless the queue was persisted.
|
|
3021
|
+
if (!this.deferredWritesPersisted && !this.quiesced) {
|
|
3022
|
+
this.unackedDeferredWrites = [];
|
|
3023
|
+
return;
|
|
3024
|
+
}
|
|
3025
|
+
try {
|
|
3026
|
+
this.store.sync();
|
|
3027
|
+
} catch (err) {
|
|
3028
|
+
console.error(
|
|
3029
|
+
`[host-mode] chronicle sync failed — ${this.unackedDeferredWrites.length} flushed deferred ` +
|
|
3030
|
+
`write(s) stay in the recovery queue until a sync succeeds:`,
|
|
3031
|
+
err,
|
|
3032
|
+
);
|
|
3033
|
+
return;
|
|
3034
|
+
}
|
|
3035
|
+
this.unackedDeferredWrites = [];
|
|
3036
|
+
this.deferredScanFrom.clear();
|
|
3037
|
+
this.persistDeferredWrites();
|
|
3038
|
+
}
|
|
3039
|
+
|
|
3040
|
+
/**
|
|
3041
|
+
* Move drained entries into the un-acked set and record, per target
|
|
3042
|
+
* agent, the store position they will be appended after — then persist
|
|
3043
|
+
* that receipt BEFORE any of them is written. Boot recovery scans from the
|
|
3044
|
+
* recorded position to the tail, so every member of an interrupted batch
|
|
3045
|
+
* is visible to the dedup, however large the batch.
|
|
3046
|
+
*/
|
|
3047
|
+
private handOffDeferredWrites(entries: DeferredWrite[]): void {
|
|
3048
|
+
if (entries.length === 0) return;
|
|
3049
|
+
const durable = this.deferredWritesPersisted || this.quiesced;
|
|
3050
|
+
if (!durable) return; // memory-only deferrals: nothing to reconcile at boot
|
|
3051
|
+
for (const entry of entries) {
|
|
3052
|
+
const target = entry.forAgent ?? this.primaryAgentName;
|
|
3053
|
+
const agent = target ? this.agents.get(target) : undefined;
|
|
3054
|
+
if (!agent) continue;
|
|
3055
|
+
let count = 0;
|
|
3056
|
+
try {
|
|
3057
|
+
const cm = agent.getContextManager() as unknown as { getMessageCount?: () => number; getAllMessages: () => unknown[] };
|
|
3058
|
+
count = typeof cm.getMessageCount === 'function' ? cm.getMessageCount() : cm.getAllMessages().length;
|
|
3059
|
+
} catch { count = 0; }
|
|
3060
|
+
const prev = this.deferredScanFrom.get(target!);
|
|
3061
|
+
this.deferredScanFrom.set(target!, prev === undefined ? count : Math.min(prev, count));
|
|
3062
|
+
}
|
|
3063
|
+
this.unackedDeferredWrites.push(...entries);
|
|
3064
|
+
this.persistDeferredWrites();
|
|
3065
|
+
}
|
|
3066
|
+
|
|
3067
|
+
/**
|
|
3068
|
+
* Which of `ids` already exist in some agent's store as a landed deferred
|
|
3069
|
+
* write (`metadata.deferredWriteId`). Scans each context manager's recent
|
|
3070
|
+
* tail blob-free — an interrupted flush is always within the last few
|
|
3071
|
+
* hundred slots — and tolerates facades without windowed reads.
|
|
3072
|
+
*/
|
|
3073
|
+
private landedDeferredWriteIds(ids: Set<string>, scanFrom: Map<string, number>): Set<string> {
|
|
3074
|
+
const landed = new Set<string>();
|
|
3075
|
+
if (ids.size === 0) return landed;
|
|
3076
|
+
const WINDOW = 500;
|
|
3077
|
+
for (const agent of this.agents.values()) {
|
|
3078
|
+
try {
|
|
3079
|
+
const cm = agent.getContextManager() as unknown as {
|
|
3080
|
+
getMessageCount?: () => number;
|
|
3081
|
+
getMessageWindow?: (o: number, l: number, opts?: { resolveBlobs?: boolean }) =>
|
|
3082
|
+
{ messages: Array<{ metadata?: Record<string, unknown> }> };
|
|
3083
|
+
getAllMessages: () => Array<{ metadata?: Record<string, unknown> }>;
|
|
3084
|
+
};
|
|
3085
|
+
const consider = (m: { metadata?: Record<string, unknown> }): void => {
|
|
3086
|
+
const id = m.metadata?.deferredWriteId;
|
|
3087
|
+
if (typeof id === 'string' && ids.has(id)) landed.add(id);
|
|
3088
|
+
};
|
|
3089
|
+
if (typeof cm.getMessageCount === 'function' && typeof cm.getMessageWindow === 'function') {
|
|
3090
|
+
const total = cm.getMessageCount();
|
|
3091
|
+
// From the receipt position (0 = whole store when unknown) to the
|
|
3092
|
+
// tail, in blob-free windows — the batch can only be after it.
|
|
3093
|
+
const from = Math.min(scanFrom.get(agent.name) ?? 0, total);
|
|
3094
|
+
for (let start = from; start < total; start += WINDOW) {
|
|
3095
|
+
const win = cm.getMessageWindow(start, Math.min(WINDOW, total - start), { resolveBlobs: false });
|
|
3096
|
+
for (const m of win.messages) consider(m);
|
|
3097
|
+
}
|
|
3098
|
+
} else {
|
|
3099
|
+
for (const m of cm.getAllMessages()) consider(m);
|
|
3100
|
+
}
|
|
3101
|
+
} catch (err) {
|
|
3102
|
+
console.error(`[host-mode] could not scan ${agent.name} for landed deferred writes:`, err);
|
|
3103
|
+
}
|
|
3104
|
+
}
|
|
3105
|
+
return landed;
|
|
3106
|
+
}
|
|
3107
|
+
|
|
3108
|
+
/** Atomic JSON write for the recovery files (tmp + rename, 0600) — the
|
|
3109
|
+
* same idiom as the Discord awareness outbox. */
|
|
3110
|
+
private writeRecoveryFile(path: string, document: unknown): void {
|
|
3111
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
3112
|
+
const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
|
|
3113
|
+
writeFileSync(temporary, `${JSON.stringify(document, null, 2)}\n`, { mode: 0o600 });
|
|
3114
|
+
renameSync(temporary, path);
|
|
3115
|
+
}
|
|
3116
|
+
|
|
3117
|
+
private readRecoveryFile(path: string): unknown {
|
|
3118
|
+
if (!existsSync(path)) return undefined;
|
|
3119
|
+
return JSON.parse(readFileSync(path, 'utf8'));
|
|
3120
|
+
}
|
|
3121
|
+
|
|
3122
|
+
private warnRecoveryFallbackOnce(): void {
|
|
3123
|
+
if (this.recoveryFallbackWarned) return;
|
|
3124
|
+
this.recoveryFallbackWarned = true;
|
|
3125
|
+
console.error(
|
|
3126
|
+
'[host-mode] no storePath / hostModePath / deferredWritesPath: quiesce state and deferred ' +
|
|
3127
|
+
'writes persist in the branch-local framework/state slot — a historical rollback while ' +
|
|
3128
|
+
'quiesced can erase them; configure the recovery paths',
|
|
3129
|
+
);
|
|
3130
|
+
}
|
|
3131
|
+
|
|
3132
|
+
/**
|
|
3133
|
+
* Mirror the deferred-write queue durably while quiesced (and until an
|
|
3134
|
+
* already-persisted queue drains). The quiesce flag itself is persisted
|
|
3135
|
+
* because a crash mid-surgery is an expected event; the messages withheld
|
|
3136
|
+
* BECAUSE of that surgery would otherwise die with the process — and the
|
|
3137
|
+
* window is operator-length, not turn-length. Lives in the
|
|
3138
|
+
* branch-independent recovery file so a rollback cannot orphan it.
|
|
3139
|
+
* Deferrals outside a quiesce window (turn-alive, mid-tool-cycle) are
|
|
3140
|
+
* still memory-only: they flush within the turn, as before.
|
|
3141
|
+
*/
|
|
3142
|
+
private persistDeferredWrites(): void {
|
|
3143
|
+
try {
|
|
3144
|
+
// The durable queue is everything not yet acked: writes still pending
|
|
3145
|
+
// AND writes handed to a context manager whose sync has not happened.
|
|
3146
|
+
// Always in original deferral order, whatever the pending/un-acked
|
|
3147
|
+
// split: a re-deferred entry must not jump the queue.
|
|
3148
|
+
const durable = [...this.unackedDeferredWrites, ...this.deferredMessages].sort(bySeq);
|
|
3149
|
+
const scanFrom = Object.fromEntries(this.deferredScanFrom);
|
|
3150
|
+
if ((this.quiesced || this.deferredWritesPersisted) && durable.length > 0) {
|
|
3151
|
+
const payload = JSON.stringify(durable);
|
|
3152
|
+
if (payload.length > DEFERRED_WRITES_PERSIST_CAP_BYTES) {
|
|
3153
|
+
if (!this.deferredWritesCapWarned) {
|
|
3154
|
+
this.deferredWritesCapWarned = true;
|
|
3155
|
+
console.error(
|
|
3156
|
+
`[host-mode] deferred context writes exceed ${DEFERRED_WRITES_PERSIST_CAP_BYTES} bytes ` +
|
|
3157
|
+
`serialized (${durable.length} message(s)) — kept in memory only; ` +
|
|
3158
|
+
`a crash before resume loses them`,
|
|
3159
|
+
);
|
|
3160
|
+
}
|
|
3161
|
+
return;
|
|
3162
|
+
}
|
|
3163
|
+
if (this.deferredWritesPath) {
|
|
3164
|
+
this.writeRecoveryFile(this.deferredWritesPath, { version: 2, pending: durable, scanFrom });
|
|
3165
|
+
} else {
|
|
3166
|
+
this.warnRecoveryFallbackOnce();
|
|
3167
|
+
this.store.setStateJson(DEFERRED_WRITES_ID, { version: 2, pending: durable, scanFrom });
|
|
3168
|
+
}
|
|
3169
|
+
this.deferredWritesPersisted = true;
|
|
3170
|
+
} else if (this.deferredWritesPersisted) {
|
|
3171
|
+
if (this.deferredWritesPath) {
|
|
3172
|
+
this.writeRecoveryFile(this.deferredWritesPath, { version: 1, pending: [] });
|
|
3173
|
+
} else {
|
|
3174
|
+
this.store.setStateJson(DEFERRED_WRITES_ID, []);
|
|
3175
|
+
}
|
|
3176
|
+
this.deferredWritesPersisted = false;
|
|
3177
|
+
}
|
|
3178
|
+
} catch (err) {
|
|
3179
|
+
console.error('[host-mode] failed to persist deferred context writes:', err);
|
|
3180
|
+
}
|
|
3181
|
+
}
|
|
3182
|
+
|
|
3183
|
+
private restorePersistedDeferredWrites(): number {
|
|
3184
|
+
try {
|
|
3185
|
+
let raw: unknown;
|
|
3186
|
+
if (this.deferredWritesPath && existsSync(this.deferredWritesPath)) {
|
|
3187
|
+
raw = this.readRecoveryFile(this.deferredWritesPath);
|
|
3188
|
+
} else {
|
|
3189
|
+
// Stores that predate the recovery file (or store-only configs).
|
|
3190
|
+
raw = this.store.getStateJson(DEFERRED_WRITES_ID);
|
|
3191
|
+
}
|
|
3192
|
+
// v1 slot payloads were a bare array; v1/v2 files and v2 slots are
|
|
3193
|
+
// `{ pending, scanFrom? }`.
|
|
3194
|
+
const doc = Array.isArray(raw) ? { pending: raw } : (raw as { pending?: unknown; scanFrom?: unknown } | undefined);
|
|
3195
|
+
const data = doc?.pending;
|
|
3196
|
+
if (!Array.isArray(data) || data.length === 0) return 0;
|
|
3197
|
+
const scanFrom = new Map<string, number>();
|
|
3198
|
+
if (doc?.scanFrom && typeof doc.scanFrom === 'object') {
|
|
3199
|
+
for (const [k, v] of Object.entries(doc.scanFrom as Record<string, unknown>)) {
|
|
3200
|
+
if (typeof v === 'number' && Number.isFinite(v)) scanFrom.set(k, Math.max(0, Math.floor(v)));
|
|
3201
|
+
}
|
|
3202
|
+
}
|
|
3203
|
+
const candidates = data
|
|
3204
|
+
.filter((m): m is Omit<DeferredWrite, 'id' | 'seq'> & { id?: string; seq?: number } =>
|
|
3205
|
+
!!m && typeof m === 'object' && typeof (m as { participant?: unknown }).participant === 'string'
|
|
3206
|
+
&& Array.isArray((m as { content?: unknown }).content))
|
|
3207
|
+
.map((m, i) => ({
|
|
3208
|
+
...m,
|
|
3209
|
+
id: typeof m.id === 'string' ? m.id : randomUUID(),
|
|
3210
|
+
// Files that predate `seq` are in write order already.
|
|
3211
|
+
seq: typeof m.seq === 'number' ? m.seq : i + 1,
|
|
3212
|
+
}))
|
|
3213
|
+
.sort(bySeq);
|
|
3214
|
+
this.deferredSeq = Math.max(this.deferredSeq, ...candidates.map((m) => m.seq));
|
|
3215
|
+
// Exactly-once: a message that landed before its ack was written is
|
|
3216
|
+
// already in the store under its deferredWriteId — do not replay it.
|
|
3217
|
+
// The scan covers every slot the interrupted batch could occupy (from
|
|
3218
|
+
// the receipt's per-agent position), or the whole store when the
|
|
3219
|
+
// receipt predates the position field.
|
|
3220
|
+
const landed = this.landedDeferredWriteIds(new Set(candidates.map((m) => m.id)), scanFrom);
|
|
3221
|
+
const restored = candidates.filter((m) => !landed.has(m.id));
|
|
3222
|
+
if (landed.size > 0) {
|
|
3223
|
+
console.error(
|
|
3224
|
+
`[host-mode] ${landed.size} deferred context write(s) had already landed before their ack ` +
|
|
3225
|
+
`(interrupted flush) — skipped, not replayed`,
|
|
3226
|
+
);
|
|
3227
|
+
}
|
|
3228
|
+
this.deferredMessages.push(...restored);
|
|
3229
|
+
this.deferredWritesPersisted = true;
|
|
3230
|
+
// Rewrite the durable queue without the already-landed entries so a
|
|
3231
|
+
// second interrupted boot does not re-scan them.
|
|
3232
|
+
if (landed.size > 0) this.persistDeferredWrites();
|
|
3233
|
+
return restored.length;
|
|
3234
|
+
} catch (err) {
|
|
3235
|
+
console.error('[host-mode] failed to restore persisted deferred context writes:', err);
|
|
3236
|
+
return 0;
|
|
3237
|
+
}
|
|
3238
|
+
}
|
|
3239
|
+
|
|
3240
|
+
/**
|
|
3241
|
+
* Operator-driven maintenance: run (or join) one bounded compression pass —
|
|
3242
|
+
* the same runQueuedMaintenance the timer drives, so tool definitions are
|
|
3243
|
+
* refreshed and ticks are bounded/serialized identically — and return the
|
|
3244
|
+
* maintenance snapshot. Works while quiesced by design: `running` stays
|
|
3245
|
+
* true in quiesce mode precisely so this machinery stays hot.
|
|
3246
|
+
*/
|
|
3247
|
+
async maintenanceTick(): Promise<ContextMaintenanceSnapshot & { ran: boolean }> {
|
|
3248
|
+
if (!this.running) {
|
|
3249
|
+
// startQueuedMaintenance no-ops on a stopped/never-started framework —
|
|
3250
|
+
// a bare snapshot would masquerade as a completed pass, so say so.
|
|
3251
|
+
console.warn('[host-mode] maintenanceTick: framework is not running — no pass executed');
|
|
3252
|
+
return { ...this.getContextMaintenanceSnapshot(), ran: false };
|
|
3253
|
+
}
|
|
3254
|
+
this.startQueuedMaintenance();
|
|
3255
|
+
await this.maintenancePass;
|
|
3256
|
+
return { ...this.getContextMaintenanceSnapshot(), ran: true };
|
|
3257
|
+
}
|
|
3258
|
+
|
|
2202
3259
|
/** Counts-only context-maintenance diagnostics for authenticated debug UIs. */
|
|
2203
3260
|
getContextMaintenanceSnapshot(): ContextMaintenanceSnapshot {
|
|
2204
3261
|
const agents = [...this.agents.values()].map((agent) => {
|
|
@@ -2342,6 +3399,101 @@ export class AgentFramework {
|
|
|
2342
3399
|
);
|
|
2343
3400
|
}
|
|
2344
3401
|
|
|
3402
|
+
/**
|
|
3403
|
+
* Feasibility preview for a runtime-settings patch — or, with no patch, for
|
|
3404
|
+
* the agent's CURRENT settings (the resume-gate case: quiesce maintenance
|
|
3405
|
+
* may have moved the folded floor and the branch generation, so the verdict
|
|
3406
|
+
* must be computed fresh at resume time).
|
|
3407
|
+
*
|
|
3408
|
+
* Models the patch through the same immediate-vs-paced semantics as the
|
|
3409
|
+
* apply path (`Agent.planRuntimeSettings`): a non-immediate budget decrease
|
|
3410
|
+
* is previewed at the UNCHANGED live compile budget, with a best-effort
|
|
3411
|
+
* advisory verdict at the descent target. This is what the superseded
|
|
3412
|
+
* `feat/budget-preflight-guard` got wrong — it previewed the patch value as
|
|
3413
|
+
* the compile budget and rejected safe-by-construction paced descents.
|
|
3414
|
+
*
|
|
3415
|
+
* Never throws for preview-layer reasons: strategy without previewContext,
|
|
3416
|
+
* preview overlap, stale branch generation, and the hierarchical (non-
|
|
3417
|
+
* adaptive) path all come back as `{available: false, reason}`. Unknown
|
|
3418
|
+
* agent still throws, matching the sibling accessors.
|
|
3419
|
+
*/
|
|
3420
|
+
previewAgentRuntimeSettings(
|
|
3421
|
+
agentName: string,
|
|
3422
|
+
patch?: AgentRuntimeSettingsPatch,
|
|
3423
|
+
): RuntimeSettingsPreview {
|
|
3424
|
+
const agent = this.agents.get(agentName);
|
|
3425
|
+
if (!agent) throw new Error(`Unknown agent: ${agentName}`);
|
|
3426
|
+
const plan = agent.planRuntimeSettings(patch);
|
|
3427
|
+
const settings = agent.getRuntimeSettings();
|
|
3428
|
+
const base: RuntimeSettingsPreview = {
|
|
3429
|
+
available: false,
|
|
3430
|
+
path: plan.path,
|
|
3431
|
+
liveBudgetTokens: plan.liveBudgetTokens,
|
|
3432
|
+
transition: settings.transition,
|
|
3433
|
+
...(settings.transitionReason ? { transitionReason: settings.transitionReason } : {}),
|
|
3434
|
+
};
|
|
3435
|
+
|
|
3436
|
+
const runPreview = (budgetTokens: number): {
|
|
3437
|
+
result?: Record<string, unknown>;
|
|
3438
|
+
reason?: string;
|
|
3439
|
+
} => {
|
|
3440
|
+
try {
|
|
3441
|
+
const result = this.previewContextSettings(
|
|
3442
|
+
agentName,
|
|
3443
|
+
budgetTokens,
|
|
3444
|
+
Object.keys(plan.overrides).length > 0 ? plan.overrides : undefined,
|
|
3445
|
+
);
|
|
3446
|
+
if (result === null || typeof result !== 'object') {
|
|
3447
|
+
return { reason: 'no_preview_support' };
|
|
3448
|
+
}
|
|
3449
|
+
return { result: result as Record<string, unknown> };
|
|
3450
|
+
} catch (error) {
|
|
3451
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3452
|
+
// Structured mapping of the strategy's known throw paths. Message
|
|
3453
|
+
// matching is fragile by nature, so unknown messages surface verbatim
|
|
3454
|
+
// as the reason rather than being swallowed or rethrown.
|
|
3455
|
+
if (message.includes('requires reinitialization for the current branch generation')) {
|
|
3456
|
+
return { reason: 'branch_generation_changed' };
|
|
3457
|
+
}
|
|
3458
|
+
if (message.includes('already running; previews must not overlap')) {
|
|
3459
|
+
return { reason: 'preview_in_flight' };
|
|
3460
|
+
}
|
|
3461
|
+
if (message.includes('requires adaptiveResolution')) {
|
|
3462
|
+
return { reason: 'no_adaptive_resolution' };
|
|
3463
|
+
}
|
|
3464
|
+
return { reason: message };
|
|
3465
|
+
}
|
|
3466
|
+
};
|
|
3467
|
+
|
|
3468
|
+
const effective = runPreview(plan.effectiveBudgetTokens);
|
|
3469
|
+
if (!effective.result) {
|
|
3470
|
+
return { ...base, reason: effective.reason };
|
|
3471
|
+
}
|
|
3472
|
+
const preview: RuntimeSettingsPreview = {
|
|
3473
|
+
...base,
|
|
3474
|
+
available: true,
|
|
3475
|
+
effective: {
|
|
3476
|
+
...effective.result,
|
|
3477
|
+
budgetTokens: plan.effectiveBudgetTokens,
|
|
3478
|
+
fits: effective.result.fits === true,
|
|
3479
|
+
finalTokens: Number(effective.result.finalTokens ?? 0),
|
|
3480
|
+
},
|
|
3481
|
+
};
|
|
3482
|
+
if (plan.advisoryTargetTokens !== undefined) {
|
|
3483
|
+
// Best-effort: an advisory failure never flips `available` — the
|
|
3484
|
+
// effective verdict above is the one that gates anything.
|
|
3485
|
+
const advisory = runPreview(plan.advisoryTargetTokens);
|
|
3486
|
+
if (advisory.result) {
|
|
3487
|
+
preview.advisory = {
|
|
3488
|
+
...advisory.result,
|
|
3489
|
+
targetTokens: plan.advisoryTargetTokens,
|
|
3490
|
+
fits: advisory.result.fits === true,
|
|
3491
|
+
};
|
|
3492
|
+
}
|
|
3493
|
+
}
|
|
3494
|
+
return preview;
|
|
3495
|
+
}
|
|
3496
|
+
|
|
2345
3497
|
/**
|
|
2346
3498
|
* Check if process logging is enabled.
|
|
2347
3499
|
*/
|
|
@@ -2525,6 +3677,75 @@ export class AgentFramework {
|
|
|
2525
3677
|
this.store.setStateJson(FRAMEWORK_STATE_ID, state);
|
|
2526
3678
|
}
|
|
2527
3679
|
|
|
3680
|
+
/** Persist (or clear, with null) the host quiesce mode. First process-global
|
|
3681
|
+
* key in the framework/state slot: quiesce survives a restart BY DESIGN — a
|
|
3682
|
+
* crash mid-surgery must come back up NOT serving against a half-repaired
|
|
3683
|
+
* store. Resume is always reachable (public method, WS/HTTP, control-plane
|
|
3684
|
+
* host/command), and every quiesced boot logs a loud banner. */
|
|
3685
|
+
private persistHostMode(mode: { quiesced: true; reason?: string; since: number } | null): void {
|
|
3686
|
+
if (this.hostModePath) {
|
|
3687
|
+
// Branch-independent: the recovery file is the record. (`quiesced:
|
|
3688
|
+
// false` is written explicitly rather than deleting the file so that
|
|
3689
|
+
// its presence, not its content, decides precedence over the legacy
|
|
3690
|
+
// branch-local slot below.)
|
|
3691
|
+
this.writeRecoveryFile(
|
|
3692
|
+
this.hostModePath,
|
|
3693
|
+
mode === null ? { version: 1, quiesced: false, clearedAt: Date.now() } : { version: 1, ...mode },
|
|
3694
|
+
);
|
|
3695
|
+
if (mode === null) {
|
|
3696
|
+
// Also drop any legacy slot marker so a downgrade cannot resurrect it.
|
|
3697
|
+
try {
|
|
3698
|
+
const data = this.store.getStateJson(FRAMEWORK_STATE_ID);
|
|
3699
|
+
if (data && typeof data === 'object' && 'hostMode' in (data as Record<string, unknown>)) {
|
|
3700
|
+
const state = { ...(data as Record<string, unknown>) };
|
|
3701
|
+
delete state.hostMode;
|
|
3702
|
+
this.store.setStateJson(FRAMEWORK_STATE_ID, state);
|
|
3703
|
+
}
|
|
3704
|
+
} catch { /* best effort */ }
|
|
3705
|
+
}
|
|
3706
|
+
} else {
|
|
3707
|
+
this.warnRecoveryFallbackOnce();
|
|
3708
|
+
const data = this.store.getStateJson(FRAMEWORK_STATE_ID);
|
|
3709
|
+
const state = (data && typeof data === 'object' ? data : {}) as Record<string, unknown>;
|
|
3710
|
+
if (mode === null) delete state.hostMode;
|
|
3711
|
+
else state.hostMode = { ...mode };
|
|
3712
|
+
this.store.setStateJson(FRAMEWORK_STATE_ID, state);
|
|
3713
|
+
}
|
|
3714
|
+
// The whole feature rests on the flag surviving a crash — close the
|
|
3715
|
+
// window between the write and the next periodic sync.
|
|
3716
|
+
try {
|
|
3717
|
+
this.store.sync();
|
|
3718
|
+
} catch (err) {
|
|
3719
|
+
console.error('[host-mode] store sync after host-mode change failed:', err);
|
|
3720
|
+
}
|
|
3721
|
+
}
|
|
3722
|
+
|
|
3723
|
+
/** The recovery file wins whenever it exists; the branch-local slot is
|
|
3724
|
+
* read only for stores that predate it (or store-only configs). */
|
|
3725
|
+
private readHostMode(): { quiesced: boolean; reason?: string; since?: number } | null {
|
|
3726
|
+
try {
|
|
3727
|
+
let record: Record<string, unknown> | null = null;
|
|
3728
|
+
if (this.hostModePath && existsSync(this.hostModePath)) {
|
|
3729
|
+
const doc = this.readRecoveryFile(this.hostModePath);
|
|
3730
|
+
if (doc && typeof doc === 'object') record = doc as Record<string, unknown>;
|
|
3731
|
+
} else {
|
|
3732
|
+
const data = this.store.getStateJson(FRAMEWORK_STATE_ID);
|
|
3733
|
+
if (!data || typeof data !== 'object') return null;
|
|
3734
|
+
const mode = (data as Record<string, unknown>).hostMode;
|
|
3735
|
+
if (!mode || typeof mode !== 'object') return null;
|
|
3736
|
+
record = mode as Record<string, unknown>;
|
|
3737
|
+
}
|
|
3738
|
+
if (!record) return null;
|
|
3739
|
+
return {
|
|
3740
|
+
quiesced: record.quiesced === true,
|
|
3741
|
+
...(typeof record.reason === 'string' ? { reason: record.reason } : {}),
|
|
3742
|
+
...(typeof record.since === 'number' ? { since: record.since } : {}),
|
|
3743
|
+
};
|
|
3744
|
+
} catch {
|
|
3745
|
+
return null;
|
|
3746
|
+
}
|
|
3747
|
+
}
|
|
3748
|
+
|
|
2528
3749
|
/**
|
|
2529
3750
|
* Create an ephemeral agent that is NOT registered in the main event loop.
|
|
2530
3751
|
*
|
|
@@ -2650,18 +3871,40 @@ export class AgentFramework {
|
|
|
2650
3871
|
contextManager: ContextManager,
|
|
2651
3872
|
watchdogs?: { startupTimeoutMs?: number; idleTimeoutMs?: number; idlePollMs?: number },
|
|
2652
3873
|
): Promise<{ speech: string; toolCallsCount: number }> {
|
|
3874
|
+
// Refused at admission while quiesced rather than requeued: the caller
|
|
3875
|
+
// awaits the settle promise under an idle-stall watchdog, so a gate-parked
|
|
3876
|
+
// ephemeral would ride out its timeout and reject with a misleading
|
|
3877
|
+
// "stalled" error hours later. A clear refusal now beats that. Checked
|
|
3878
|
+
// before the generation ticket is consumed so the same (agent, cm) pair
|
|
3879
|
+
// can be retried after resume().
|
|
3880
|
+
if (this.quiesced) {
|
|
3881
|
+
throw new Error(
|
|
3882
|
+
`framework is quiesced${this.quiesceReason ? ` (${this.quiesceReason})` : ''}: ` +
|
|
3883
|
+
`refusing new ephemeral run for ${agent.name} — resume() first`,
|
|
3884
|
+
);
|
|
3885
|
+
}
|
|
2653
3886
|
// Only a fresh object returned by createEphemeralAgent may enter this path.
|
|
2654
3887
|
// Never overwrite a resident/conversation owner or a concurrent run: their
|
|
2655
3888
|
// cleanup is name-keyed and could cancel/deregister the legitimate owner.
|
|
2656
3889
|
if (this.ephemeralCandidates.get(agent) !== contextManager) {
|
|
2657
3890
|
throw new Error(`Ephemeral agent "${agent.name}" has no fresh generation ticket from this framework`);
|
|
2658
3891
|
}
|
|
3892
|
+
// A live surgery holds the whole store: an agent admitted now would
|
|
3893
|
+
// compile against a history that is being replaced under it. Refuse
|
|
3894
|
+
// BEFORE consuming the ticket so the caller can retry once it completes.
|
|
3895
|
+
if (this.surgeryHold) {
|
|
3896
|
+
throw new Error(
|
|
3897
|
+
`Ephemeral agent "${agent.name}" refused: the store is under live ${this.surgeryHold.verb} ` +
|
|
3898
|
+
`for ${this.surgeryHold.agentName} — retry when it completes`,
|
|
3899
|
+
);
|
|
3900
|
+
}
|
|
2659
3901
|
// Consume before any await/registration: the exact (Agent, ContextManager)
|
|
2660
3902
|
// generation is one-shot even if startup later fails.
|
|
2661
3903
|
this.ephemeralCandidates.delete(agent);
|
|
2662
3904
|
if (this.agents.has(agent.name) || this.ephemeralRuns.has(agent.name)) {
|
|
2663
3905
|
throw new Error(`Ephemeral agent "${agent.name}" is already registered or running`);
|
|
2664
3906
|
}
|
|
3907
|
+
// Register temporarily so the event loop can drive it
|
|
2665
3908
|
this.agents.set(agent.name, agent);
|
|
2666
3909
|
const run: EphemeralRun = {
|
|
2667
3910
|
settle: this.createDeferred<AgentSettleResult>(),
|
|
@@ -3406,6 +4649,8 @@ export class AgentFramework {
|
|
|
3406
4649
|
/** Agent state at queue time — 'idle' means the turn starts on the next
|
|
3407
4650
|
* scheduler pass; anything else means it runs after the current turn. */
|
|
3408
4651
|
agentStatus?: string;
|
|
4652
|
+
/** True when the host is quiesced: the wake is PARKED, not running, until resume(). */
|
|
4653
|
+
quiesced?: boolean;
|
|
3409
4654
|
} {
|
|
3410
4655
|
const name = agentName ?? [...this.agents.keys()][0];
|
|
3411
4656
|
const agent = name ? this.agents.get(name) : undefined;
|
|
@@ -3421,8 +4666,17 @@ export class AgentFramework {
|
|
|
3421
4666
|
source: 'admin',
|
|
3422
4667
|
timestamp: Date.now(),
|
|
3423
4668
|
});
|
|
3424
|
-
console.error(
|
|
3425
|
-
|
|
4669
|
+
console.error(
|
|
4670
|
+
`[nudge] agent=${name} queued by=${requestedBy ?? 'unknown'} status=${agentStatus}` +
|
|
4671
|
+
(this.quiesced ? ' (host quiesced — parked until resume)' : ''),
|
|
4672
|
+
);
|
|
4673
|
+
this.recordOperatorAction({
|
|
4674
|
+
kind: 'nudge',
|
|
4675
|
+
agent: name,
|
|
4676
|
+
requester: { via: 'nudge', ...(requestedBy ? { name: requestedBy } : {}) },
|
|
4677
|
+
result: { agentStatus, ...(this.quiesced ? { quiesced: true } : {}) },
|
|
4678
|
+
});
|
|
4679
|
+
return { ok: true, agentName: name, agentStatus, ...(this.quiesced ? { quiesced: true } : {}) };
|
|
3426
4680
|
}
|
|
3427
4681
|
|
|
3428
4682
|
/**
|
|
@@ -3436,6 +4690,20 @@ export class AgentFramework {
|
|
|
3436
4690
|
*
|
|
3437
4691
|
* nudge — run inference on the current context with NO new events
|
|
3438
4692
|
* (see `nudgeAgent`).
|
|
4693
|
+
*
|
|
4694
|
+
* quiesce — enter host maintenance mode (issue #122): park wakes, pause
|
|
4695
|
+
* MCPL data planes, drain in-flight turns (optionally `abandon` on
|
|
4696
|
+
* timeout). Host-scoped, no agentName.
|
|
4697
|
+
*
|
|
4698
|
+
* resume — leave maintenance mode behind a fresh feasibility preview of
|
|
4699
|
+
* every agent's current settings (`force` overrides a failing verdict).
|
|
4700
|
+
*
|
|
4701
|
+
* maintain — run (or join) one bounded compression/maintenance pass and
|
|
4702
|
+
* return the snapshot; the operator lever for draining quarantine or
|
|
4703
|
+
* advancing merges during a quiesce window.
|
|
4704
|
+
*
|
|
4705
|
+
* host-status — report the quiesce state (drained / parked wakes /
|
|
4706
|
+
* running background scripts).
|
|
3439
4707
|
*/
|
|
3440
4708
|
private async handleHostCommand(
|
|
3441
4709
|
serverId: string,
|
|
@@ -3443,8 +4711,14 @@ export class AgentFramework {
|
|
|
3443
4711
|
): Promise<{
|
|
3444
4712
|
ok: boolean;
|
|
3445
4713
|
error?: string;
|
|
4714
|
+
/** Refusal code from live surgery (e.g. 'agent-busy'), when applicable. */
|
|
4715
|
+
code?: string;
|
|
3446
4716
|
undone?: number;
|
|
3447
4717
|
requested?: number;
|
|
4718
|
+
/** For quiesce/resume/maintain/host-status: the host mode snapshot. */
|
|
4719
|
+
hostMode?: HostModeStatus;
|
|
4720
|
+
/** For `maintain`: the context-maintenance snapshot after the pass. */
|
|
4721
|
+
maintenance?: ContextMaintenanceSnapshot;
|
|
3448
4722
|
messagesRemoved?: number;
|
|
3449
4723
|
/** Discord addresses removed by message-granular undo. The durable outbox
|
|
3450
4724
|
* owns eventual delivery; this is also returned for immediate surfaces. */
|
|
@@ -3458,6 +4732,9 @@ export class AgentFramework {
|
|
|
3458
4732
|
* outcome report is posted to the channel asynchronously. */
|
|
3459
4733
|
started?: boolean;
|
|
3460
4734
|
cap?: number;
|
|
4735
|
+
/** For `nudge`/`unstick`: the host is quiesced, so the request is parked
|
|
4736
|
+
* until resume rather than running. */
|
|
4737
|
+
quiesced?: boolean;
|
|
3461
4738
|
/** For `nudge`: agent state at queue time ('idle' = runs immediately,
|
|
3462
4739
|
* else it runs once the current turn settles). */
|
|
3463
4740
|
agentStatus?: string;
|
|
@@ -3466,11 +4743,48 @@ export class AgentFramework {
|
|
|
3466
4743
|
params.command !== 'undo' &&
|
|
3467
4744
|
params.command !== 'hide' &&
|
|
3468
4745
|
params.command !== 'unstick' &&
|
|
3469
|
-
params.command !== 'nudge'
|
|
4746
|
+
params.command !== 'nudge' &&
|
|
4747
|
+
params.command !== 'quiesce' &&
|
|
4748
|
+
params.command !== 'resume' &&
|
|
4749
|
+
params.command !== 'maintain' &&
|
|
4750
|
+
params.command !== 'host-status'
|
|
3470
4751
|
) {
|
|
3471
4752
|
return { ok: false, error: `Unknown host command: ${String(params.command)}` };
|
|
3472
4753
|
}
|
|
3473
4754
|
|
|
4755
|
+
// Host-scoped verbs (issue #122) — no agent resolution.
|
|
4756
|
+
const requester = params.requesterName ?? params.requesterId ?? `mcpl:${serverId}`;
|
|
4757
|
+
if (params.command === 'quiesce') {
|
|
4758
|
+
// quiesce() clamps timeoutMs to [1s, 10m] itself — one clamp for every ingress.
|
|
4759
|
+
console.error(`[host-command] quiesce by=${requester} (server=${serverId})`);
|
|
4760
|
+
const hostMode = await this.quiesce({
|
|
4761
|
+
reason: params.reason ?? `host/command by ${requester}`,
|
|
4762
|
+
...(params.timeoutMs !== undefined ? { timeoutMs: params.timeoutMs } : {}),
|
|
4763
|
+
...(params.abandon ? { abandon: true } : {}),
|
|
4764
|
+
});
|
|
4765
|
+
return { ok: true, hostMode };
|
|
4766
|
+
}
|
|
4767
|
+
if (params.command === 'resume') {
|
|
4768
|
+
console.error(`[host-command] resume by=${requester} (server=${serverId})`);
|
|
4769
|
+
try {
|
|
4770
|
+
const hostMode = await this.resume(params.force ? { force: true } : undefined);
|
|
4771
|
+
return { ok: true, hostMode };
|
|
4772
|
+
} catch (error) {
|
|
4773
|
+
if (error instanceof ResumeBlockedError) {
|
|
4774
|
+
return { ok: false, error: error.message, hostMode: this.getHostModeStatus() };
|
|
4775
|
+
}
|
|
4776
|
+
throw error;
|
|
4777
|
+
}
|
|
4778
|
+
}
|
|
4779
|
+
if (params.command === 'maintain') {
|
|
4780
|
+
console.error(`[host-command] maintain by=${requester} (server=${serverId})`);
|
|
4781
|
+
const maintenance = await this.maintenanceTick();
|
|
4782
|
+
return { ok: true, maintenance, hostMode: this.getHostModeStatus() };
|
|
4783
|
+
}
|
|
4784
|
+
if (params.command === 'host-status') {
|
|
4785
|
+
return { ok: true, hostMode: this.getHostModeStatus() };
|
|
4786
|
+
}
|
|
4787
|
+
|
|
3474
4788
|
const agentName = params.agentName ?? [...this.agents.keys()][0];
|
|
3475
4789
|
if (!agentName || !this.agents.has(agentName)) {
|
|
3476
4790
|
return { ok: false, error: `Unknown agent: ${String(agentName)}` };
|
|
@@ -3483,7 +4797,7 @@ export class AgentFramework {
|
|
|
3483
4797
|
agentName,
|
|
3484
4798
|
params.requesterName ?? params.requesterId ?? `mcpl:${serverId}`,
|
|
3485
4799
|
);
|
|
3486
|
-
return { ok: r.ok, error: r.error, agentStatus: r.agentStatus };
|
|
4800
|
+
return { ok: r.ok, error: r.error, agentStatus: r.agentStatus, ...(r.quiesced ? { quiesced: true } : {}) };
|
|
3487
4801
|
}
|
|
3488
4802
|
|
|
3489
4803
|
// unstick: force the refusal-rewind loop on demand (even if the agent's
|
|
@@ -3515,7 +4829,14 @@ export class AgentFramework {
|
|
|
3515
4829
|
`[unstick] agent=${agentName} started cap=${cap} ` +
|
|
3516
4830
|
`by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`,
|
|
3517
4831
|
);
|
|
3518
|
-
|
|
4832
|
+
this.recordOperatorAction({
|
|
4833
|
+
kind: 'unstick',
|
|
4834
|
+
agent: agentName,
|
|
4835
|
+
requester: hostCommandRequester(serverId, params),
|
|
4836
|
+
params: { cap },
|
|
4837
|
+
result: { started: true, ...(this.quiesced ? { quiesced: true } : {}) },
|
|
4838
|
+
});
|
|
4839
|
+
return { ok: true, started: true, cap, ...(this.quiesced ? { quiesced: true } : {}) };
|
|
3519
4840
|
}
|
|
3520
4841
|
|
|
3521
4842
|
// hide: redact a single message (or an inclusive range) from the active
|
|
@@ -3565,11 +4886,25 @@ export class AgentFramework {
|
|
|
3565
4886
|
}
|
|
3566
4887
|
const [lo, hi] = fromIdx <= toIdx ? [fromIdx, toIdx] : [toIdx, fromIdx];
|
|
3567
4888
|
const refs = refsIn(lo, hi);
|
|
4889
|
+
const rangeIds = all.slice(lo, hi + 1).map((m) => String(m.id));
|
|
3568
4890
|
cm.removeMessages(all[lo].id, all[hi].id);
|
|
3569
4891
|
console.error(
|
|
3570
4892
|
`[host-command] hide agent=${agentName} range removed=${hi - lo + 1} ` +
|
|
3571
4893
|
`(${params.fromMessageId}..${params.toMessageId}) by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`,
|
|
3572
4894
|
);
|
|
4895
|
+
const loggedRange = capIds(rangeIds);
|
|
4896
|
+
this.recordOperatorAction({
|
|
4897
|
+
kind: 'hide',
|
|
4898
|
+
agent: agentName,
|
|
4899
|
+
requester: hostCommandRequester(serverId, params),
|
|
4900
|
+
params: { fromMessageId: params.fromMessageId, toMessageId: params.toMessageId },
|
|
4901
|
+
result: {
|
|
4902
|
+
branch: this.store.currentBranch().name,
|
|
4903
|
+
hidden: hi - lo + 1,
|
|
4904
|
+
removedIds: loggedRange.ids,
|
|
4905
|
+
...(loggedRange.truncated ? { removedIdsTruncated: true } : {}),
|
|
4906
|
+
},
|
|
4907
|
+
});
|
|
3573
4908
|
return {
|
|
3574
4909
|
ok: true,
|
|
3575
4910
|
hidden: hi - lo + 1,
|
|
@@ -3578,11 +4913,19 @@ export class AgentFramework {
|
|
|
3578
4913
|
};
|
|
3579
4914
|
}
|
|
3580
4915
|
const refs = refsIn(fromIdx, fromIdx);
|
|
4916
|
+
const hiddenId = String(all[fromIdx].id);
|
|
3581
4917
|
cm.removeMessage(all[fromIdx].id);
|
|
3582
4918
|
console.error(
|
|
3583
4919
|
`[host-command] hide agent=${agentName} removed=1 (${params.fromMessageId}) ` +
|
|
3584
4920
|
`by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`,
|
|
3585
4921
|
);
|
|
4922
|
+
this.recordOperatorAction({
|
|
4923
|
+
kind: 'hide',
|
|
4924
|
+
agent: agentName,
|
|
4925
|
+
requester: hostCommandRequester(serverId, params),
|
|
4926
|
+
params: { fromMessageId: params.fromMessageId },
|
|
4927
|
+
result: { branch: this.store.currentBranch().name, hidden: 1, removedIds: [hiddenId] },
|
|
4928
|
+
});
|
|
3586
4929
|
return {
|
|
3587
4930
|
ok: true,
|
|
3588
4931
|
hidden: 1,
|
|
@@ -3605,62 +4948,40 @@ export class AgentFramework {
|
|
|
3605
4948
|
}
|
|
3606
4949
|
const n = Math.max(1, Math.min(50, Math.floor(params.messages)));
|
|
3607
4950
|
const cm = agent.getContextManager();
|
|
3608
|
-
const
|
|
3609
|
-
if (n >=
|
|
4951
|
+
const total = cm.getMessageCount();
|
|
4952
|
+
if (n >= total) {
|
|
3610
4953
|
return {
|
|
3611
4954
|
ok: false,
|
|
3612
|
-
error: `Cannot remove ${n} message(s) — history has ${
|
|
4955
|
+
error: `Cannot remove ${n} message(s) — history has ${total}; at least one must remain.`,
|
|
3613
4956
|
};
|
|
3614
4957
|
}
|
|
3615
|
-
const target =
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
|
|
3622
|
-
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
|
|
3626
|
-
|
|
3627
|
-
|
|
3628
|
-
|
|
3629
|
-
|
|
3630
|
-
|
|
3631
|
-
|
|
3632
|
-
|
|
3633
|
-
|
|
3634
|
-
if (markerBatch) this.discordAwarenessOutbox!.activate(markerBatch.id);
|
|
3635
|
-
await this.syncDiscordAwarenessMarkers();
|
|
3636
|
-
|
|
3637
|
-
// Materialize config files from the new branch (fire-and-forget; gate
|
|
3638
|
-
// picks up via mtime) — mirrors undoLastTurn.
|
|
3639
|
-
const wsUndo = this.moduleRegistry.getModule('workspace');
|
|
3640
|
-
if (wsUndo && 'materializeMount' in wsUndo) {
|
|
3641
|
-
(wsUndo as { materializeMount: (m: string) => Promise<void> })
|
|
3642
|
-
.materializeMount('_config')
|
|
3643
|
-
.catch(() => {});
|
|
4958
|
+
const target = cm.getMessageWindow(total - 1 - n, 1, { resolveBlobs: false }).messages[0];
|
|
4959
|
+
try {
|
|
4960
|
+
const r = await this.rollbackToMessage(agentName, {
|
|
4961
|
+
messageId: String(target.id),
|
|
4962
|
+
branchName: `undo-msgs/${agentName}/${Date.now()}`,
|
|
4963
|
+
requester: hostCommandRequester(serverId, params),
|
|
4964
|
+
});
|
|
4965
|
+
return {
|
|
4966
|
+
ok: true,
|
|
4967
|
+
messagesRemoved: r.messagesRemoved,
|
|
4968
|
+
removedRefs: r.removedRefs,
|
|
4969
|
+
lastVisible: r.lastVisible,
|
|
4970
|
+
};
|
|
4971
|
+
} catch (error) {
|
|
4972
|
+
return {
|
|
4973
|
+
ok: false,
|
|
4974
|
+
error: error instanceof Error ? error.message : String(error),
|
|
4975
|
+
...(error instanceof OperatorActionError ? { code: error.code } : {}),
|
|
4976
|
+
};
|
|
3644
4977
|
}
|
|
3645
|
-
|
|
3646
|
-
console.error(
|
|
3647
|
-
`[host-command] undo-messages agent=${agentName} removed=${n} branch=${branchName}` +
|
|
3648
|
-
` by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`,
|
|
3649
|
-
);
|
|
3650
|
-
|
|
3651
|
-
return {
|
|
3652
|
-
ok: true,
|
|
3653
|
-
messagesRemoved: n,
|
|
3654
|
-
removedRefs,
|
|
3655
|
-
lastVisible: await this.lastVisiblePreview(agentName),
|
|
3656
|
-
};
|
|
3657
4978
|
}
|
|
3658
4979
|
|
|
3659
4980
|
const requested = Math.max(1, Math.min(20, Math.floor(params.turns ?? 1)));
|
|
3660
4981
|
let undone = 0;
|
|
3661
4982
|
try {
|
|
3662
4983
|
for (let i = 0; i < requested; i++) {
|
|
3663
|
-
const r = this.undoLastTurn(agentName);
|
|
4984
|
+
const r = this.undoLastTurn(agentName, hostCommandRequester(serverId, params));
|
|
3664
4985
|
if (!r.undone) break;
|
|
3665
4986
|
undone++;
|
|
3666
4987
|
}
|
|
@@ -3998,7 +5319,7 @@ export class AgentFramework {
|
|
|
3998
5319
|
* The undone branch is saved so `redo()` can restore it.
|
|
3999
5320
|
* Returns the checkpoint that was undone, or null if nothing to undo.
|
|
4000
5321
|
*/
|
|
4001
|
-
undoLastTurn(agentName: string): {
|
|
5322
|
+
undoLastTurn(agentName: string, requester?: OperatorRequester): {
|
|
4002
5323
|
undone: boolean;
|
|
4003
5324
|
turnIndex?: number;
|
|
4004
5325
|
fromBranch?: string;
|
|
@@ -4047,6 +5368,13 @@ export class AgentFramework {
|
|
|
4047
5368
|
fromBranch: currentBranch.name,
|
|
4048
5369
|
toBranch: undoBranchName,
|
|
4049
5370
|
});
|
|
5371
|
+
this.recordOperatorAction({
|
|
5372
|
+
kind: 'undo-turn',
|
|
5373
|
+
agent: agentName,
|
|
5374
|
+
...(requester ? { requester } : {}),
|
|
5375
|
+
params: { turnIndex: checkpoint.turnIndex },
|
|
5376
|
+
result: { sourceBranch: currentBranch.name, targetBranch: undoBranchName },
|
|
5377
|
+
});
|
|
4050
5378
|
|
|
4051
5379
|
return {
|
|
4052
5380
|
undone: true,
|
|
@@ -4062,7 +5390,7 @@ export class AgentFramework {
|
|
|
4062
5390
|
* Switches back to the branch that was active before the last undo.
|
|
4063
5391
|
* Returns false if there's nothing to redo.
|
|
4064
5392
|
*/
|
|
4065
|
-
redo(agentName: string): {
|
|
5393
|
+
redo(agentName: string, requester?: OperatorRequester): {
|
|
4066
5394
|
redone: boolean;
|
|
4067
5395
|
fromBranch?: string;
|
|
4068
5396
|
toBranch?: string;
|
|
@@ -4102,6 +5430,12 @@ export class AgentFramework {
|
|
|
4102
5430
|
fromBranch: currentBranch.name,
|
|
4103
5431
|
toBranch: branchName,
|
|
4104
5432
|
});
|
|
5433
|
+
this.recordOperatorAction({
|
|
5434
|
+
kind: 'redo-turn',
|
|
5435
|
+
agent: agentName,
|
|
5436
|
+
...(requester ? { requester } : {}),
|
|
5437
|
+
result: { sourceBranch: currentBranch.name, targetBranch: branchName },
|
|
5438
|
+
});
|
|
4105
5439
|
|
|
4106
5440
|
return {
|
|
4107
5441
|
redone: true,
|
|
@@ -4110,6 +5444,472 @@ export class AgentFramework {
|
|
|
4110
5444
|
};
|
|
4111
5445
|
}
|
|
4112
5446
|
|
|
5447
|
+
// ---------------------------------------------------------------------------
|
|
5448
|
+
// Live operator surgery — rollback / suppress on the OPEN store, without a
|
|
5449
|
+
// restart — plus the durable action log every operator mutation writes to.
|
|
5450
|
+
//
|
|
5451
|
+
// Both follow the same idiom the offline surgeries use (recovery/
|
|
5452
|
+
// offline-branch.ts): fork first, mutate the fork, and make the fork the
|
|
5453
|
+
// active branch. The parent branch keeps everything, so restore is a
|
|
5454
|
+
// checkout. Mutations are refused (never queued) while the agent is not
|
|
5455
|
+
// idle — quiesce the host first if it is busy.
|
|
5456
|
+
// ---------------------------------------------------------------------------
|
|
5457
|
+
|
|
5458
|
+
/**
|
|
5459
|
+
* Append a record to the operator action log and broadcast it as an
|
|
5460
|
+
* `operator:action` trace. Hosts use this directly for actions the
|
|
5461
|
+
* framework does not own (e.g. a WebUI toggling quiesce). Never throws.
|
|
5462
|
+
*/
|
|
5463
|
+
recordOperatorAction(input: OperatorLogInput): OperatorLogEntry {
|
|
5464
|
+
// Observability never fails the action: OperatorLog.append never throws,
|
|
5465
|
+
// and a misbehaving trace listener is contained here.
|
|
5466
|
+
const entry = this.operatorLog.append(input);
|
|
5467
|
+
try {
|
|
5468
|
+
this.emitTrace({
|
|
5469
|
+
type: 'operator:action',
|
|
5470
|
+
kind: entry.kind,
|
|
5471
|
+
...(entry.agent !== undefined ? { agentName: entry.agent } : {}),
|
|
5472
|
+
...(entry.requester !== undefined ? { requester: entry.requester } : {}),
|
|
5473
|
+
...(entry.note !== undefined ? { note: entry.note } : {}),
|
|
5474
|
+
...(entry.params !== undefined ? { params: entry.params } : {}),
|
|
5475
|
+
...(entry.result !== undefined ? { result: entry.result } : {}),
|
|
5476
|
+
...(entry.error !== undefined ? { error: entry.error } : {}),
|
|
5477
|
+
});
|
|
5478
|
+
} catch {
|
|
5479
|
+
// swallowed by design
|
|
5480
|
+
}
|
|
5481
|
+
return entry;
|
|
5482
|
+
}
|
|
5483
|
+
|
|
5484
|
+
/** Newest `limit` operator-log entries, oldest first. */
|
|
5485
|
+
getOperatorLog(opts?: { limit?: number }): OperatorLogEntry[] {
|
|
5486
|
+
return this.operatorLog.readTail(opts?.limit ?? 100);
|
|
5487
|
+
}
|
|
5488
|
+
|
|
5489
|
+
/** Where the operator log is written, or undefined when disabled. */
|
|
5490
|
+
getOperatorLogPath(): string | undefined {
|
|
5491
|
+
return this.operatorLog.path;
|
|
5492
|
+
}
|
|
5493
|
+
|
|
5494
|
+
/**
|
|
5495
|
+
* The surgery gate — STORE-wide, not per agent. A branch switch moves the
|
|
5496
|
+
* active chronicle branch for every context manager sharing this store,
|
|
5497
|
+
* so an in-flight turn on *any* agent would have its next writes land on
|
|
5498
|
+
* a history its request was never compiled against. Every agent must be
|
|
5499
|
+
* idle with no turn alive (`state.status === 'idle'` alone is not enough:
|
|
5500
|
+
* status reads idle from dequeue until the stream registers and again
|
|
5501
|
+
* during teardown — the scheduler's own busy test is status + turn-alive,
|
|
5502
|
+
* the predicate `puppetToolCall` uses).
|
|
5503
|
+
*
|
|
5504
|
+
* Passing the gate RESERVES the store for the caller: a turn token is
|
|
5505
|
+
* held for every agent until the returned release runs, so no wake — via
|
|
5506
|
+
* the scheduler or a parked provider admission, both of which re-test
|
|
5507
|
+
* turn-alive — can start a turn while the switch is awaited. Release also
|
|
5508
|
+
* flushes messages that deferred behind the reservation, as the puppet
|
|
5509
|
+
* does. Always release in `finally`: a token nobody clears wedges the fleet.
|
|
5510
|
+
*/
|
|
5511
|
+
private reserveStoreForSurgery(verb: string, agentName: string): () => void {
|
|
5512
|
+
if (this.surgeryHold) {
|
|
5513
|
+
throw new OperatorActionError(
|
|
5514
|
+
'agent-busy',
|
|
5515
|
+
`Cannot ${verb}: a live ${this.surgeryHold.verb} for ${this.surgeryHold.agentName} is already in progress`,
|
|
5516
|
+
);
|
|
5517
|
+
}
|
|
5518
|
+
const busy: string[] = [];
|
|
5519
|
+
for (const [name, a] of this.agents) {
|
|
5520
|
+
if (a.state.status !== 'idle') busy.push(`${name} is ${a.state.status}`);
|
|
5521
|
+
else if (this.activeTurnTokens.has(name)) busy.push(`${name} is idle+turn-alive`);
|
|
5522
|
+
}
|
|
5523
|
+
for (const name of this.activeTurnTokens.keys()) {
|
|
5524
|
+
if (!this.agents.has(name)) busy.push(`${name} is turn-alive`);
|
|
5525
|
+
}
|
|
5526
|
+
if (busy.length > 0) {
|
|
5527
|
+
throw new OperatorActionError(
|
|
5528
|
+
'agent-busy',
|
|
5529
|
+
`Cannot ${verb} while ${busy.join(', ')} — every agent sharing the store must be idle (quiesce the host first)`,
|
|
5530
|
+
);
|
|
5531
|
+
}
|
|
5532
|
+
const reserved = new Map<string, number>();
|
|
5533
|
+
for (const name of this.agents.keys()) {
|
|
5534
|
+
const token = this.nextTurnToken++;
|
|
5535
|
+
this.activeTurnTokens.set(name, token);
|
|
5536
|
+
reserved.set(name, token);
|
|
5537
|
+
}
|
|
5538
|
+
// The token snapshot only covers agents that exist now; the hold covers
|
|
5539
|
+
// arrivals during the awaited switch (see surgeryHold).
|
|
5540
|
+
const hold = { verb, agentName, since: Date.now() };
|
|
5541
|
+
this.surgeryHold = hold;
|
|
5542
|
+
let released = false;
|
|
5543
|
+
return () => {
|
|
5544
|
+
if (released) return;
|
|
5545
|
+
released = true;
|
|
5546
|
+
if (this.surgeryHold === hold) this.surgeryHold = null;
|
|
5547
|
+
for (const [name, token] of reserved) {
|
|
5548
|
+
if (this.activeTurnTokens.get(name) === token) this.activeTurnTokens.delete(name);
|
|
5549
|
+
}
|
|
5550
|
+
// Writers that deferred behind the reservation land now, on the branch
|
|
5551
|
+
// the operator chose (or the restored source) — the same end-of-turn
|
|
5552
|
+
// flush driveStream/puppet perform, under the same guard.
|
|
5553
|
+
if (this.deferredMessages.length > 0 && this.pendingAssistantBlocks.size === 0) {
|
|
5554
|
+
for (const name of reserved.keys()) {
|
|
5555
|
+
if (this.activeTurnTokens.has(name)) continue;
|
|
5556
|
+
for (const msg of this.drainDeferredFor(name)) {
|
|
5557
|
+
try {
|
|
5558
|
+
this.addMessage(msg.participant, msg.content, msg.metadata,
|
|
5559
|
+
msg.forAgent ? { forAgent: msg.forAgent } : undefined);
|
|
5560
|
+
} catch (error) {
|
|
5561
|
+
console.error(`[operator] post-surgery deferred flush failed for ${name}: ${error instanceof Error ? error.message : String(error)}`);
|
|
5562
|
+
}
|
|
5563
|
+
}
|
|
5564
|
+
}
|
|
5565
|
+
}
|
|
5566
|
+
};
|
|
5567
|
+
}
|
|
5568
|
+
|
|
5569
|
+
/**
|
|
5570
|
+
* After a failed switch or redaction: put the store back on `sourceBranch`
|
|
5571
|
+
* if it moved, and return a sentence saying what actually happened. Never
|
|
5572
|
+
* throws — the caller reports the original failure with this appended.
|
|
5573
|
+
*/
|
|
5574
|
+
private async restoreSourceBranch(
|
|
5575
|
+
cm: ContextManager,
|
|
5576
|
+
sourceBranch: string,
|
|
5577
|
+
failedBranch: string,
|
|
5578
|
+
agentName: string,
|
|
5579
|
+
): Promise<string> {
|
|
5580
|
+
const current = this.store.currentBranch().name;
|
|
5581
|
+
if (current !== failedBranch) return `active branch is ${current}`;
|
|
5582
|
+
try {
|
|
5583
|
+
await cm.switchBranch(sourceBranch);
|
|
5584
|
+
return `active branch restored to ${sourceBranch}`;
|
|
5585
|
+
} catch (restoreError) {
|
|
5586
|
+
const detail = restoreError instanceof Error ? restoreError.message : String(restoreError);
|
|
5587
|
+
const status = `RESTORE TO ${sourceBranch} ALSO FAILED (${detail}); agent is still on ${failedBranch}`;
|
|
5588
|
+
console.error(`[operator] agent=${agentName} ${status}`);
|
|
5589
|
+
return status;
|
|
5590
|
+
}
|
|
5591
|
+
}
|
|
5592
|
+
|
|
5593
|
+
/**
|
|
5594
|
+
* Roll the active branch back so `messageId` becomes its tail: fork the
|
|
5595
|
+
* chronicle at that message (origin-sequence time-travel branch) and switch
|
|
5596
|
+
* to the fork. Everything after the message stays on the source branch.
|
|
5597
|
+
* Discord messages that left the live context get awareness markers via the
|
|
5598
|
+
* durable outbox, exactly as message-granular `undo` does.
|
|
5599
|
+
*
|
|
5600
|
+
* Throws `OperatorActionError` (`agent-busy`, `unknown-message`, …) — the
|
|
5601
|
+
* agent must be idle; nothing is queued.
|
|
5602
|
+
*/
|
|
5603
|
+
async rollbackToMessage(
|
|
5604
|
+
agentName: string,
|
|
5605
|
+
opts: {
|
|
5606
|
+
messageId: string;
|
|
5607
|
+
requester?: OperatorRequester;
|
|
5608
|
+
note?: string;
|
|
5609
|
+
/** Branch name for the fork (default `rollback/<agent>/<ts>`). */
|
|
5610
|
+
branchName?: string;
|
|
5611
|
+
},
|
|
5612
|
+
): Promise<{
|
|
5613
|
+
agentName: string;
|
|
5614
|
+
sourceBranch: string;
|
|
5615
|
+
targetBranch: string;
|
|
5616
|
+
messagesRemoved: number;
|
|
5617
|
+
/** The message that actually became the tail. Differs from the requested
|
|
5618
|
+
* id only when that id was a shard of a body group: the fork lands on
|
|
5619
|
+
* the group's last shard so the sharded message stays whole. */
|
|
5620
|
+
tailMessageId: string;
|
|
5621
|
+
removedRefs: Array<{ serverId: string; channelId: string; messageId: string }>;
|
|
5622
|
+
lastVisible: { participant?: string; role?: string; preview?: string } | null;
|
|
5623
|
+
}> {
|
|
5624
|
+
const logBase: OperatorLogInput = {
|
|
5625
|
+
kind: 'rollback',
|
|
5626
|
+
agent: agentName,
|
|
5627
|
+
...(opts.requester ? { requester: opts.requester } : {}),
|
|
5628
|
+
...(opts.note ? { note: opts.note } : {}),
|
|
5629
|
+
params: { messageId: opts.messageId },
|
|
5630
|
+
};
|
|
5631
|
+
try {
|
|
5632
|
+
const agent = this.agents.get(agentName);
|
|
5633
|
+
if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${agentName}`);
|
|
5634
|
+
const cm = agent.getContextManager();
|
|
5635
|
+
const total = cm.getMessageCount();
|
|
5636
|
+
const requestedIndex = locateMessageIndex(cm, opts.messageId);
|
|
5637
|
+
if (requestedIndex < 0) {
|
|
5638
|
+
throw new OperatorActionError('unknown-message', `Message ${opts.messageId} is not on the active branch`);
|
|
5639
|
+
}
|
|
5640
|
+
// Never bisect a body group: chronicle refuses that for removals
|
|
5641
|
+
// (byte-faithful reassembly needs the whole run) and branchAt has no
|
|
5642
|
+
// such guard of its own. Snap to the run's last shard.
|
|
5643
|
+
const run = bodyGroupRun(cm, requestedIndex);
|
|
5644
|
+
const targetIndex = run.to;
|
|
5645
|
+
const tailMessageId = run.toId;
|
|
5646
|
+
const messagesRemoved = total - targetIndex - 1;
|
|
5647
|
+
if (messagesRemoved <= 0) {
|
|
5648
|
+
throw new OperatorActionError('invalid', `Message ${opts.messageId} is already the tail of the active branch`);
|
|
5649
|
+
}
|
|
5650
|
+
const discarded = cm.getMessageWindow(targetIndex + 1, messagesRemoved, { resolveBlobs: false }).messages;
|
|
5651
|
+
const removedRefs = extractDiscordAwarenessRefs(discarded);
|
|
5652
|
+
const sourceBranch = this.store.currentBranch().name;
|
|
5653
|
+
const targetBranch = opts.branchName ?? `rollback/${agentName}/${Date.now()}`;
|
|
5654
|
+
if (targetBranch === sourceBranch) {
|
|
5655
|
+
throw new OperatorActionError('invalid', 'Rollback branch name must differ from the active branch');
|
|
5656
|
+
}
|
|
5657
|
+
|
|
5658
|
+
// Gate + reserve the whole store (see reserveStoreForSurgery); held
|
|
5659
|
+
// until the switch has landed or been rolled back.
|
|
5660
|
+
const release = this.reserveStoreForSurgery('roll back', agentName);
|
|
5661
|
+
let branchName: string;
|
|
5662
|
+
try {
|
|
5663
|
+
// Prepare the external side effect before switching Chronicle. If the
|
|
5664
|
+
// process dies after the switch but before activate(), startup
|
|
5665
|
+
// promotes this batch by matching targetBranch to the active branch.
|
|
5666
|
+
// If the fork/switch itself throws, the batch is retired so it cannot
|
|
5667
|
+
// arm a later branch that happens to reuse the name.
|
|
5668
|
+
const markerBatch = this.discordAwarenessOutbox?.prepare({
|
|
5669
|
+
agentName,
|
|
5670
|
+
sourceBranch,
|
|
5671
|
+
targetBranch,
|
|
5672
|
+
refs: removedRefs,
|
|
5673
|
+
emoji: this.discordAwarenessEmoji,
|
|
5674
|
+
}) ?? null;
|
|
5675
|
+
|
|
5676
|
+
try {
|
|
5677
|
+
branchName = cm.branchAt(tailMessageId as MessageId, targetBranch);
|
|
5678
|
+
await cm.switchBranch(branchName);
|
|
5679
|
+
} catch (error) {
|
|
5680
|
+
// switchBranch moves the chronicle branch BEFORE awaiting strategy
|
|
5681
|
+
// initialization, so a rejection here can leave the store on the
|
|
5682
|
+
// new branch with an uninitialized strategy. Retire the batch and
|
|
5683
|
+
// go back to the source; keep the branch for diagnosis.
|
|
5684
|
+
if (markerBatch) this.discordAwarenessOutbox!.discard(markerBatch.id);
|
|
5685
|
+
const restored = await this.restoreSourceBranch(cm, sourceBranch, targetBranch, agentName);
|
|
5686
|
+
throw new OperatorActionError(
|
|
5687
|
+
'failed',
|
|
5688
|
+
`Rollback failed; ${restored}: ${error instanceof Error ? error.message : String(error)}`,
|
|
5689
|
+
{ cause: error },
|
|
5690
|
+
);
|
|
5691
|
+
}
|
|
5692
|
+
if (markerBatch) this.discordAwarenessOutbox!.activate(markerBatch.id);
|
|
5693
|
+
await this.syncDiscordAwarenessMarkers();
|
|
5694
|
+
this.materializeConfigMountAfterBranchSwitch();
|
|
5695
|
+
} finally {
|
|
5696
|
+
release();
|
|
5697
|
+
}
|
|
5698
|
+
|
|
5699
|
+
console.error(
|
|
5700
|
+
`[operator] rollback agent=${agentName} to=${tailMessageId}` +
|
|
5701
|
+
`${tailMessageId !== opts.messageId ? ` (requested ${opts.messageId}, snapped to end of body group)` : ''}` +
|
|
5702
|
+
` removed=${messagesRemoved} branch=${branchName} by=${describeRequester(opts.requester)}`,
|
|
5703
|
+
);
|
|
5704
|
+
this.recordOperatorAction({
|
|
5705
|
+
...logBase,
|
|
5706
|
+
result: {
|
|
5707
|
+
sourceBranch,
|
|
5708
|
+
targetBranch: branchName,
|
|
5709
|
+
tailMessageId,
|
|
5710
|
+
messagesRemoved,
|
|
5711
|
+
discordRefs: removedRefs.length,
|
|
5712
|
+
},
|
|
5713
|
+
});
|
|
5714
|
+
return {
|
|
5715
|
+
agentName,
|
|
5716
|
+
sourceBranch,
|
|
5717
|
+
targetBranch: branchName,
|
|
5718
|
+
messagesRemoved,
|
|
5719
|
+
tailMessageId,
|
|
5720
|
+
removedRefs,
|
|
5721
|
+
lastVisible: await this.lastVisiblePreview(agentName),
|
|
5722
|
+
};
|
|
5723
|
+
} catch (error) {
|
|
5724
|
+
this.recordOperatorAction({ ...logBase, error: error instanceof Error ? error.message : String(error) });
|
|
5725
|
+
throw error;
|
|
5726
|
+
}
|
|
5727
|
+
}
|
|
5728
|
+
|
|
5729
|
+
/**
|
|
5730
|
+
* Suppress specific messages from the live context without touching the
|
|
5731
|
+
* source branch: fork at the current head (`suppress/<agent>/<ts>`), redact
|
|
5732
|
+
* the messages on the fork, switch to it. Shards of a body group are always
|
|
5733
|
+
* removed together. Discord originals get awareness markers via the outbox;
|
|
5734
|
+
* the batch is `explicit`-activated only after every removal succeeded, and
|
|
5735
|
+
* a crash mid-way is finished at next boot (resumePreparedDiscordSuppressions).
|
|
5736
|
+
*
|
|
5737
|
+
* Not retroactive over derived state: a message already folded into an
|
|
5738
|
+
* autobiographical summary stays in that summary — roll back to before it
|
|
5739
|
+
* entered if that matters.
|
|
5740
|
+
*/
|
|
5741
|
+
async suppressMessages(
|
|
5742
|
+
agentName: string,
|
|
5743
|
+
opts: {
|
|
5744
|
+
messageIds: string[];
|
|
5745
|
+
requester?: OperatorRequester;
|
|
5746
|
+
note?: string;
|
|
5747
|
+
/** Branch name for the fork (default `suppress/<agent>/<ts>`). */
|
|
5748
|
+
branchName?: string;
|
|
5749
|
+
},
|
|
5750
|
+
): Promise<{
|
|
5751
|
+
agentName: string;
|
|
5752
|
+
sourceBranch: string;
|
|
5753
|
+
targetBranch: string;
|
|
5754
|
+
messagesRemoved: number;
|
|
5755
|
+
removedIds: string[];
|
|
5756
|
+
removedRefs: Array<{ serverId: string; channelId: string; messageId: string }>;
|
|
5757
|
+
lastVisible: { participant?: string; role?: string; preview?: string } | null;
|
|
5758
|
+
}> {
|
|
5759
|
+
const requestedIds = [...new Set(opts.messageIds.map(String))];
|
|
5760
|
+
const logBase: OperatorLogInput = {
|
|
5761
|
+
kind: 'suppress',
|
|
5762
|
+
agent: agentName,
|
|
5763
|
+
...(opts.requester ? { requester: opts.requester } : {}),
|
|
5764
|
+
...(opts.note ? { note: opts.note } : {}),
|
|
5765
|
+
params: { messageIds: requestedIds },
|
|
5766
|
+
};
|
|
5767
|
+
try {
|
|
5768
|
+
const agent = this.agents.get(agentName);
|
|
5769
|
+
if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${agentName}`);
|
|
5770
|
+
if (requestedIds.length === 0) throw new OperatorActionError('invalid', 'No message ids given');
|
|
5771
|
+
const cm = agent.getContextManager();
|
|
5772
|
+
const total = cm.getMessageCount();
|
|
5773
|
+
|
|
5774
|
+
const located = locateMessageIndices(cm, new Set(requestedIds));
|
|
5775
|
+
const missing = requestedIds.filter((id) => !located.has(id));
|
|
5776
|
+
if (missing.length > 0) {
|
|
5777
|
+
throw new OperatorActionError(
|
|
5778
|
+
'unknown-message',
|
|
5779
|
+
`Not on the active branch: ${missing.slice(0, 5).join(', ')}${missing.length > 5 ? ` (+${missing.length - 5})` : ''}`,
|
|
5780
|
+
);
|
|
5781
|
+
}
|
|
5782
|
+
|
|
5783
|
+
// Build removal intervals (index-ordered, newest first). A shard expands
|
|
5784
|
+
// to its whole body group — chronicle refuses to bisect one — and a
|
|
5785
|
+
// group interval is always removed as a RANGE: `removeMessage` refuses
|
|
5786
|
+
// any sharded message even when the group is down to one shard.
|
|
5787
|
+
const intervals = new Map<string, ReturnType<typeof bodyGroupRun>>();
|
|
5788
|
+
for (const id of requestedIds) {
|
|
5789
|
+
const run = bodyGroupRun(cm, located.get(id)!.index);
|
|
5790
|
+
intervals.set(`${run.from}-${run.to}`, run);
|
|
5791
|
+
}
|
|
5792
|
+
const ordered = [...intervals.values()].sort((a, b) => b.from - a.from);
|
|
5793
|
+
const messagesRemoved = ordered.reduce((n, iv) => n + (iv.to - iv.from + 1), 0);
|
|
5794
|
+
if (messagesRemoved >= total) {
|
|
5795
|
+
throw new OperatorActionError('invalid', `Cannot suppress all ${total} message(s) — at least one must remain`);
|
|
5796
|
+
}
|
|
5797
|
+
const targeted = ordered.flatMap((iv) =>
|
|
5798
|
+
cm.getMessageWindow(iv.from, iv.to - iv.from + 1, { resolveBlobs: false }).messages,
|
|
5799
|
+
);
|
|
5800
|
+
const removedIds = targeted.map((m) => String(m.id));
|
|
5801
|
+
const removedRefs = extractDiscordAwarenessRefs(targeted);
|
|
5802
|
+
|
|
5803
|
+
const sourceBranch = this.store.currentBranch().name;
|
|
5804
|
+
const targetBranch = opts.branchName ?? `suppress/${agentName}/${Date.now()}`;
|
|
5805
|
+
if (targetBranch === sourceBranch) {
|
|
5806
|
+
throw new OperatorActionError('invalid', 'Suppression branch name must differ from the active branch');
|
|
5807
|
+
}
|
|
5808
|
+
// Gate + reserve the whole store (see reserveStoreForSurgery); held
|
|
5809
|
+
// until the fork is fully redacted or the source is restored.
|
|
5810
|
+
const release = this.reserveStoreForSurgery('suppress', agentName);
|
|
5811
|
+
let createdBranch: string;
|
|
5812
|
+
try {
|
|
5813
|
+
const markerBatch = this.discordAwarenessOutbox?.prepare({
|
|
5814
|
+
agentName,
|
|
5815
|
+
sourceBranch,
|
|
5816
|
+
targetBranch,
|
|
5817
|
+
refs: removedRefs,
|
|
5818
|
+
emoji: this.discordAwarenessEmoji,
|
|
5819
|
+
// Seeing targetBranch active does not prove the removals finished;
|
|
5820
|
+
// only this operation activates the batch, after the last redaction.
|
|
5821
|
+
activationPolicy: 'explicit',
|
|
5822
|
+
suppressionIntervals: ordered.map((iv) => ({ fromId: iv.fromId, toId: iv.toId })),
|
|
5823
|
+
}) ?? null;
|
|
5824
|
+
// Anything short of activation retires the batch: a prepared explicit
|
|
5825
|
+
// batch re-arms through preparedSuppressionsForBranch the moment its
|
|
5826
|
+
// target branch (or a descendant) becomes active — e.g. an operator
|
|
5827
|
+
// opening the failed fork to look — and an un-completable resume
|
|
5828
|
+
// there aborts framework start.
|
|
5829
|
+
const retireBatch = (): void => {
|
|
5830
|
+
if (markerBatch) this.discordAwarenessOutbox!.discard(markerBatch.id);
|
|
5831
|
+
};
|
|
5832
|
+
|
|
5833
|
+
try {
|
|
5834
|
+
createdBranch = await cm.fork(targetBranch);
|
|
5835
|
+
} catch (error) {
|
|
5836
|
+
// fork() switches the chronicle branch before awaiting strategy
|
|
5837
|
+
// initialization; a rejection can leave the store on the new
|
|
5838
|
+
// branch. Retire the batch and go back to the source.
|
|
5839
|
+
retireBatch();
|
|
5840
|
+
const restored = await this.restoreSourceBranch(cm, sourceBranch, targetBranch, agentName);
|
|
5841
|
+
throw new OperatorActionError(
|
|
5842
|
+
'failed',
|
|
5843
|
+
`Suppression failed creating ${targetBranch}; ${restored}: ${error instanceof Error ? error.message : String(error)}`,
|
|
5844
|
+
{ cause: error },
|
|
5845
|
+
);
|
|
5846
|
+
}
|
|
5847
|
+
try {
|
|
5848
|
+
for (const iv of ordered) {
|
|
5849
|
+
if (iv.group || iv.fromId !== iv.toId) cm.removeMessages(iv.fromId as MessageId, iv.toId as MessageId);
|
|
5850
|
+
else cm.removeMessage(iv.fromId as MessageId);
|
|
5851
|
+
}
|
|
5852
|
+
if (markerBatch) this.discordAwarenessOutbox!.activate(markerBatch.id);
|
|
5853
|
+
} catch (error) {
|
|
5854
|
+
// A partially suppressed branch is not safe to serve from. Keep it
|
|
5855
|
+
// for diagnosis, put the agent back on the untouched source — and
|
|
5856
|
+
// say honestly whether that restore happened.
|
|
5857
|
+
retireBatch();
|
|
5858
|
+
const restored = await this.restoreSourceBranch(cm, sourceBranch, createdBranch, agentName);
|
|
5859
|
+
throw new OperatorActionError(
|
|
5860
|
+
'failed',
|
|
5861
|
+
`Suppression failed on ${createdBranch}; ${restored}: ${error instanceof Error ? error.message : String(error)}`,
|
|
5862
|
+
{ cause: error },
|
|
5863
|
+
);
|
|
5864
|
+
}
|
|
5865
|
+
await this.syncDiscordAwarenessMarkers();
|
|
5866
|
+
this.materializeConfigMountAfterBranchSwitch();
|
|
5867
|
+
} finally {
|
|
5868
|
+
release();
|
|
5869
|
+
}
|
|
5870
|
+
|
|
5871
|
+
console.error(
|
|
5872
|
+
`[operator] suppress agent=${agentName} removed=${messagesRemoved} branch=${createdBranch} ` +
|
|
5873
|
+
`by=${describeRequester(opts.requester)}`,
|
|
5874
|
+
);
|
|
5875
|
+
const loggedIds = capIds(removedIds);
|
|
5876
|
+
this.recordOperatorAction({
|
|
5877
|
+
...logBase,
|
|
5878
|
+
result: {
|
|
5879
|
+
sourceBranch,
|
|
5880
|
+
targetBranch: createdBranch,
|
|
5881
|
+
messagesRemoved,
|
|
5882
|
+
removedIds: loggedIds.ids,
|
|
5883
|
+
...(loggedIds.truncated ? { removedIdsTruncated: true } : {}),
|
|
5884
|
+
discordRefs: removedRefs.length,
|
|
5885
|
+
},
|
|
5886
|
+
});
|
|
5887
|
+
return {
|
|
5888
|
+
agentName,
|
|
5889
|
+
sourceBranch,
|
|
5890
|
+
targetBranch: createdBranch,
|
|
5891
|
+
messagesRemoved,
|
|
5892
|
+
removedIds,
|
|
5893
|
+
removedRefs,
|
|
5894
|
+
lastVisible: await this.lastVisiblePreview(agentName),
|
|
5895
|
+
};
|
|
5896
|
+
} catch (error) {
|
|
5897
|
+
this.recordOperatorAction({ ...logBase, error: error instanceof Error ? error.message : String(error) });
|
|
5898
|
+
throw error;
|
|
5899
|
+
}
|
|
5900
|
+
}
|
|
5901
|
+
|
|
5902
|
+
/** Re-materialize config files from the (new) active branch — fire and
|
|
5903
|
+
* forget; the gate picks changes up via mtime. Mirrors undoLastTurn. */
|
|
5904
|
+
private materializeConfigMountAfterBranchSwitch(): void {
|
|
5905
|
+
const ws = this.moduleRegistry.getModule('workspace');
|
|
5906
|
+
if (ws && 'materializeMount' in ws) {
|
|
5907
|
+
(ws as { materializeMount: (m: string) => Promise<void> })
|
|
5908
|
+
.materializeMount('_config')
|
|
5909
|
+
.catch(() => {});
|
|
5910
|
+
}
|
|
5911
|
+
}
|
|
5912
|
+
|
|
4113
5913
|
/**
|
|
4114
5914
|
* Check if undo/redo is available for an agent.
|
|
4115
5915
|
*/
|
|
@@ -4252,7 +6052,12 @@ export class AgentFramework {
|
|
|
4252
6052
|
!this.queue.isEmpty ||
|
|
4253
6053
|
// Direct inference requests (e.g. runEphemeralToCompletion) bypass the
|
|
4254
6054
|
// event queue — without this the loop can exit before they're drained.
|
|
4255
|
-
|
|
6055
|
+
// While quiesced, gate-parked requests are NOT progress and must not
|
|
6056
|
+
// block idle (only a budget restart can still run); identity-equivalent
|
|
6057
|
+
// to `.length > 0` when not quiesced.
|
|
6058
|
+
this.pendingRequests.some(
|
|
6059
|
+
(r) => !this.quiesced || isTurnContinuation(r.reason),
|
|
6060
|
+
) ||
|
|
4256
6061
|
this.activeStreams.size > 0 ||
|
|
4257
6062
|
Array.from(this.agents.values()).some((a) => a.state.status !== 'idle')
|
|
4258
6063
|
) {
|
|
@@ -4506,7 +6311,7 @@ export class AgentFramework {
|
|
|
4506
6311
|
const deferred = this.drainDeferredFor(agent.name);
|
|
4507
6312
|
if (deferred.length > 0) {
|
|
4508
6313
|
for (const msg of deferred) {
|
|
4509
|
-
agent.getContextManager().addMessage(msg.participant, msg.content, msg.metadata);
|
|
6314
|
+
agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id));
|
|
4510
6315
|
// Injection guards: tool blocks would corrupt the tool-cycle
|
|
4511
6316
|
// structure the membrane enforces, and a message named as the
|
|
4512
6317
|
// agent itself would render as an ASSISTANT turn on the wire
|
|
@@ -4526,6 +6331,10 @@ export class AgentFramework {
|
|
|
4526
6331
|
}
|
|
4527
6332
|
}
|
|
4528
6333
|
|
|
6334
|
+
// Deferred writes handed to the store above: sync, then drop them
|
|
6335
|
+
// from the durable recovery queue (no-op unless one is persisted).
|
|
6336
|
+
this.ackDeferredWrites();
|
|
6337
|
+
|
|
4529
6338
|
// A newly injected CONVERSATIONAL message begins a new
|
|
4530
6339
|
// conversational round inside the same provider inference. Remember
|
|
4531
6340
|
// that boundary for driveStream: an explicit send in the preceding
|
|
@@ -5549,16 +7358,61 @@ export class AgentFramework {
|
|
|
5549
7358
|
// This mirrors how the restart has always overwritten activeStreams
|
|
5550
7359
|
// rather than waiting for the old stream's teardown.
|
|
5551
7360
|
const budgetRestart = requests.find((r) => r.reason === 'context_budget_restart');
|
|
7361
|
+
|
|
7362
|
+
// Host quiesce (issue #122): park wakes instead of starting turns —
|
|
7363
|
+
// the requeue mirrors the busy path below, so requests survive and
|
|
7364
|
+
// fire at resume. A context-budget restart passes through: it
|
|
7365
|
+
// CONTINUES a turn whose token is held (see the deadlock note below),
|
|
7366
|
+
// and quiesce's drain is exactly the phase where such a turn must be
|
|
7367
|
+
// allowed to finish. Parked requests are coalesced per reason (newest
|
|
7368
|
+
// kept): heartbeat/gate wakes arrive once per tick and nothing else
|
|
7369
|
+
// bounds a multi-day maintenance window's accumulation.
|
|
7370
|
+
if (this.quiesced) {
|
|
7371
|
+
// Continuations of a held turn (budget restart, non-streaming tool
|
|
7372
|
+
// results) pass; everything else parks.
|
|
7373
|
+
const passThrough = requests.filter((r) => isTurnContinuation(r.reason));
|
|
7374
|
+
const parked = requests.filter((r) => !isTurnContinuation(r.reason));
|
|
7375
|
+
// Coalesce per (reason, addressed): every channel wake shares one
|
|
7376
|
+
// reason, so keying on reason alone would let an ambient message
|
|
7377
|
+
// parked later evict a DM/mention parked earlier — and the
|
|
7378
|
+
// addressed-outranks-ambient trigger selection below would then
|
|
7379
|
+
// route the resumed turn into the wrong channel.
|
|
7380
|
+
const newestByKey = new Map<string, InferenceRequest>();
|
|
7381
|
+
for (const request of parked) {
|
|
7382
|
+
const key = `${request.reason}|${request.addressed ? 'a' : 'n'}`;
|
|
7383
|
+
const prev = newestByKey.get(key);
|
|
7384
|
+
if (!prev || request.timestamp >= prev.timestamp) {
|
|
7385
|
+
newestByKey.set(key, request);
|
|
7386
|
+
}
|
|
7387
|
+
}
|
|
7388
|
+
this.pendingRequests.push(...newestByKey.values());
|
|
7389
|
+
if (passThrough.length === 0) continue;
|
|
7390
|
+
// The continuation continues its held turn; its parked siblings must
|
|
7391
|
+
// NOT be consumed by that turn — they were re-parked above, so narrow
|
|
7392
|
+
// the batch to the continuation(s) alone before the trigger selection.
|
|
7393
|
+
requests.length = 0;
|
|
7394
|
+
requests.push(...passThrough);
|
|
7395
|
+
}
|
|
7396
|
+
|
|
5552
7397
|
const turnAlive = !budgetRestart && this.activeTurnTokens.has(agentName);
|
|
5553
7398
|
const providerGate = this.providerGates?.get(agentName);
|
|
5554
7399
|
// A primary can own provider admission while yielding to an already in-flight
|
|
5555
7400
|
// auxiliary call. Keep this agent's later wakes queued, but do not block the
|
|
5556
7401
|
// framework event loop or other residents while that auxiliary call settles.
|
|
5557
7402
|
const providerPrimaryWaiting = (providerGate?.primaryDepth ?? 0) > 0 && !this.activeTurnTokens.has(agentName);
|
|
5558
|
-
|
|
5559
|
-
|
|
7403
|
+
// A live surgery (rollback/suppress) holds the whole store: no turn may
|
|
7404
|
+
// start for ANY agent — including one registered after the reservation's
|
|
7405
|
+
// token snapshot — until the switch has landed or been rolled back.
|
|
7406
|
+
// Truthiness, not `!== null`: prototype-built harnesses (and any field
|
|
7407
|
+
// added later) leave the slot undefined, which must read as "not held".
|
|
7408
|
+
const surgeryHeld = !!this.surgeryHold;
|
|
7409
|
+
if (surgeryHeld || providerPrimaryWaiting || turnAlive || agent.state.status === 'inferring' || agent.state.status === 'streaming' || agent.state.status === 'waiting_for_tools') {
|
|
7410
|
+
// Re-queue requests, but warn if they've been pending too long.
|
|
7411
|
+
// Suppressed while quiesced: a drain-phase busy requeue is expected,
|
|
7412
|
+
// not a wedge tell.
|
|
5560
7413
|
const oldest = Math.min(...requests.map(r => r.timestamp));
|
|
5561
7414
|
if (
|
|
7415
|
+
!this.quiesced &&
|
|
5562
7416
|
now - oldest > STALE_REQUEST_MS &&
|
|
5563
7417
|
(this.staleWarnAt.get(agentName) ?? 0) < now - 60_000
|
|
5564
7418
|
) {
|
|
@@ -5571,9 +7425,11 @@ export class AgentFramework {
|
|
|
5571
7425
|
// read 'idle' while a turn's teardown is pending, and a LEAKED turn
|
|
5572
7426
|
// token would look exactly like this — permanently requeued wakes.
|
|
5573
7427
|
// This line is the wedge's tell (idle+turn-alive, forever).
|
|
5574
|
-
const shownStatus =
|
|
5575
|
-
?
|
|
5576
|
-
: agent.state.status
|
|
7428
|
+
const shownStatus = surgeryHeld && agent.state.status === 'idle'
|
|
7429
|
+
? `idle+surgery(${this.surgeryHold!.verb})`
|
|
7430
|
+
: turnAlive && agent.state.status === 'idle'
|
|
7431
|
+
? 'idle+turn-alive'
|
|
7432
|
+
: agent.state.status;
|
|
5577
7433
|
this.emitTrace({
|
|
5578
7434
|
type: 'inference:request_stale',
|
|
5579
7435
|
agentName,
|
|
@@ -6132,6 +7988,37 @@ export class AgentFramework {
|
|
|
6132
7988
|
this.releasePrimaryProviderGate(agent.name);
|
|
6133
7989
|
return;
|
|
6134
7990
|
}
|
|
7991
|
+
// The scheduler's turn-alive test passed BEFORE the park; the wait
|
|
7992
|
+
// is the one gap in which another turn-alive holder can take the
|
|
7993
|
+
// agent (puppetToolCall's reservation, #145). Never replace its
|
|
7994
|
+
// token — the re-entry would set ours over it, and a short turn
|
|
7995
|
+
// here ends and flushes before the puppeted tool returns, leaving
|
|
7996
|
+
// its pair queued behind a flush that already happened. Give
|
|
7997
|
+
// admission back and hand the wake to the scheduler, which keeps
|
|
7998
|
+
// it queued while the agent is turn-alive and starts it normally
|
|
7999
|
+
// once the holder releases.
|
|
8000
|
+
if (this.activeTurnTokens.has(agent.name)) {
|
|
8001
|
+
this.releasePrimaryProviderGate(agent.name);
|
|
8002
|
+
this.pendingRequests.push(trigger ?? {
|
|
8003
|
+
agentName: agent.name, reason: 'provider-admission:requeue', source: 'scheduler', timestamp: Date.now(),
|
|
8004
|
+
});
|
|
8005
|
+
console.error(`[provider-admission] ${agent.name}: turn-alive while parked — wake requeued, not started`);
|
|
8006
|
+
return;
|
|
8007
|
+
}
|
|
8008
|
+
// Same gap, second hazard (#122): the host may have QUIESCED while
|
|
8009
|
+
// this wake was parked. The scheduler's quiesce gate ran before the
|
|
8010
|
+
// park; starting here would bypass it and run a turn against a
|
|
8011
|
+
// context the operator believes frozen. Give admission back and
|
|
8012
|
+
// requeue — the scheduler re-parks it (coalesced) until resume.
|
|
8013
|
+
// Continuations of a held turn pass, as they do at the gate.
|
|
8014
|
+
if (this.quiesced && !isTurnContinuation(trigger?.reason ?? '')) {
|
|
8015
|
+
this.releasePrimaryProviderGate(agent.name);
|
|
8016
|
+
this.pendingRequests.push(trigger ?? {
|
|
8017
|
+
agentName: agent.name, reason: 'provider-admission:requeue', source: 'scheduler', timestamp: Date.now(),
|
|
8018
|
+
});
|
|
8019
|
+
console.error(`[provider-admission] ${agent.name}: host quiesced while parked — wake requeued, not started`);
|
|
8020
|
+
return;
|
|
8021
|
+
}
|
|
6135
8022
|
await this.startAgentStream(agent, trigger, attempt, true);
|
|
6136
8023
|
}).catch((error) => {
|
|
6137
8024
|
this.releasePrimaryProviderGate(agent.name);
|
|
@@ -6213,7 +8100,9 @@ export class AgentFramework {
|
|
|
6213
8100
|
// poison message — or a transient store-write failure — must not
|
|
6214
8101
|
// abort the turn or drop the messages behind it in the queue.
|
|
6215
8102
|
try {
|
|
6216
|
-
const id = agent.getContextManager().addMessage(
|
|
8103
|
+
const id = agent.getContextManager().addMessage(
|
|
8104
|
+
msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id),
|
|
8105
|
+
);
|
|
6217
8106
|
this.emitTrace({ type: 'message:added', messageId: id, source: 'deferred-flush:turn-start' });
|
|
6218
8107
|
} catch (err) {
|
|
6219
8108
|
console.error(
|
|
@@ -6223,6 +8112,7 @@ export class AgentFramework {
|
|
|
6223
8112
|
);
|
|
6224
8113
|
}
|
|
6225
8114
|
}
|
|
8115
|
+
this.ackDeferredWrites();
|
|
6226
8116
|
}
|
|
6227
8117
|
}
|
|
6228
8118
|
|
|
@@ -7398,6 +9288,41 @@ export class AgentFramework {
|
|
|
7398
9288
|
this.abortAgentScript(agent.name, 'stream error');
|
|
7399
9289
|
agent.reset();
|
|
7400
9290
|
|
|
9291
|
+
// A quiesce-abandoned cancel may surface as `error` instead of
|
|
9292
|
+
// `aborted` depending on how the stream implementation reports
|
|
9293
|
+
// the cancellation. Same contract as the aborted branch: settle
|
|
9294
|
+
// honestly, no errorPolicy retry (which would relaunch inference
|
|
9295
|
+
// mid-maintenance-window), no inference:exhausted (which feeds
|
|
9296
|
+
// the failure streak / hard-down / poison-history accounting).
|
|
9297
|
+
{
|
|
9298
|
+
const cancelKey = `${agent.name}:${myStreamId}`;
|
|
9299
|
+
if (this.frameworkCancelledStreams.get(cancelKey) === 'quiesce_abandoned') {
|
|
9300
|
+
this.frameworkCancelledStreams.delete(cancelKey);
|
|
9301
|
+
// Same terminal as the `aborted` twin: §10.5 lifecycle reads
|
|
9302
|
+
// 'aborted' (not 'completed'), and the gate is released only
|
|
9303
|
+
// for THIS agent instance — a name re-registered meanwhile
|
|
9304
|
+
// (ephemeral disposal, conversation replacement) owns its own
|
|
9305
|
+
// gate liveness. The inference-log terminal was already
|
|
9306
|
+
// written at the top of this case (`Stream error`).
|
|
9307
|
+
lifecyclePhase = 'aborted';
|
|
9308
|
+
this.settleAgent(agent.name, {
|
|
9309
|
+
stopReason: 'exhausted',
|
|
9310
|
+
speech: '',
|
|
9311
|
+
error: 'Turn abandoned by operator quiesce',
|
|
9312
|
+
});
|
|
9313
|
+
this.emitTrace({
|
|
9314
|
+
type: 'inference:aborted',
|
|
9315
|
+
agentName: agent.name,
|
|
9316
|
+
durationMs,
|
|
9317
|
+
reason: 'quiesce_abandoned',
|
|
9318
|
+
});
|
|
9319
|
+
if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) {
|
|
9320
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
9321
|
+
}
|
|
9322
|
+
break;
|
|
9323
|
+
}
|
|
9324
|
+
}
|
|
9325
|
+
|
|
7401
9326
|
if (ownsProviderGate && this.holdProviderAcceleration(agent, err, trigger)) {
|
|
7402
9327
|
lifecyclePhase = 'failed';
|
|
7403
9328
|
if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) this.eventGate?.onInferenceEnded(agent.name);
|
|
@@ -7453,6 +9378,30 @@ export class AgentFramework {
|
|
|
7453
9378
|
} else if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) {
|
|
7454
9379
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
7455
9380
|
}
|
|
9381
|
+
if (cancelKind === 'quiesce_abandoned') {
|
|
9382
|
+
// Operator-initiated cancel during a quiesce drain. Settle
|
|
9383
|
+
// the turn honestly — deliberately NO inference:exhausted:
|
|
9384
|
+
// that trace centrally drives the consecutive-failure
|
|
9385
|
+
// streak, hard-down ops alerts, and the poison-history
|
|
9386
|
+
// breaker, none of which an operator cancel represents.
|
|
9387
|
+
// inference:aborted carries the observability instead.
|
|
9388
|
+
if (agent.streamId === myStreamId) {
|
|
9389
|
+
this.abortAgentScript(agent.name, 'turn abandoned by quiesce');
|
|
9390
|
+
agent.reset();
|
|
9391
|
+
this.settleAgent(agent.name, {
|
|
9392
|
+
stopReason: 'exhausted',
|
|
9393
|
+
speech: '',
|
|
9394
|
+
error: 'Turn abandoned by operator quiesce',
|
|
9395
|
+
});
|
|
9396
|
+
}
|
|
9397
|
+
this.emitTrace({
|
|
9398
|
+
type: 'inference:aborted',
|
|
9399
|
+
agentName: agent.name,
|
|
9400
|
+
durationMs: Date.now() - startTime,
|
|
9401
|
+
reason: 'quiesce_abandoned',
|
|
9402
|
+
});
|
|
9403
|
+
return;
|
|
9404
|
+
}
|
|
7456
9405
|
// endTurn IS a logical turn end — earlier rounds may have
|
|
7457
9406
|
// live-routed prose (narrate → skip_reply is a real shape),
|
|
7458
9407
|
// so settle the delivery chain and drop the receipt. A
|
|
@@ -7744,9 +9693,12 @@ export class AgentFramework {
|
|
|
7744
9693
|
if (frameReachedTerminal && this.deferredMessages.length > 0 && this.pendingAssistantBlocks.size === 0) {
|
|
7745
9694
|
const deferred = this.drainDeferredFor(agent.name);
|
|
7746
9695
|
for (const msg of deferred) {
|
|
7747
|
-
this.addMessage(msg.participant, msg.content, msg.metadata,
|
|
7748
|
-
|
|
9696
|
+
this.addMessage(msg.participant, msg.content, msg.metadata, {
|
|
9697
|
+
deferredWriteId: msg.id,
|
|
9698
|
+
...(msg.forAgent ? { forAgent: msg.forAgent } : {}),
|
|
9699
|
+
});
|
|
7749
9700
|
}
|
|
9701
|
+
this.ackDeferredWrites();
|
|
7750
9702
|
}
|
|
7751
9703
|
}
|
|
7752
9704
|
}
|
|
@@ -7786,8 +9738,11 @@ export class AgentFramework {
|
|
|
7786
9738
|
* affordance to a model that cannot find it (older models especially).
|
|
7787
9739
|
*
|
|
7788
9740
|
* Semantics:
|
|
7789
|
-
* - Refused unless the agent is idle
|
|
7790
|
-
* the turn state machine and the live stream's
|
|
9741
|
+
* - Refused unless the agent is idle with no turn alive: puppeting
|
|
9742
|
+
* mid-turn would corrupt the turn state machine and the live stream's
|
|
9743
|
+
* wire ordering. For its own duration the puppet HOLDS the turn-alive
|
|
9744
|
+
* marker (wakes requeue, cross-turn writers defer), so a wake landing
|
|
9745
|
+
* during the tool's execution cannot start a turn underneath the pair.
|
|
7791
9746
|
* - Refused for tools outside the agent's own surface (canUseTool): the
|
|
7792
9747
|
* stored pair must be an act the agent could genuinely have taken.
|
|
7793
9748
|
* - The call executes FOR REAL through the shared dispatch (MCPL,
|
|
@@ -7811,9 +9766,18 @@ export class AgentFramework {
|
|
|
7811
9766
|
): Promise<{ toolUseId: string; result: ToolResult }> {
|
|
7812
9767
|
const agent = this.agents.get(agentName);
|
|
7813
9768
|
if (!agent) throw new Error(`Unknown agent: ${agentName}`);
|
|
7814
|
-
|
|
9769
|
+
// Idle AND no turn alive: status reads 'idle' from dequeue until the
|
|
9770
|
+
// stream registers, and again while a turn's teardown is pending
|
|
9771
|
+
// (the scheduler's own busy test, 'idle+turn-alive').
|
|
9772
|
+
if (this.surgeryHold) {
|
|
9773
|
+
throw new Error(
|
|
9774
|
+
`puppet refused: the store is under live ${this.surgeryHold.verb} for ${this.surgeryHold.agentName} — retry when it completes`,
|
|
9775
|
+
);
|
|
9776
|
+
}
|
|
9777
|
+
if (agent.state.status !== 'idle' || this.activeTurnTokens.has(agentName)) {
|
|
9778
|
+
const shown = agent.state.status === 'idle' ? 'idle+turn-alive' : agent.state.status;
|
|
7815
9779
|
throw new Error(
|
|
7816
|
-
`puppet refused: agent ${agentName} is ${
|
|
9780
|
+
`puppet refused: agent ${agentName} is ${shown} (requires idle — ` +
|
|
7817
9781
|
`injecting a turn under an active stream corrupts wire ordering)`,
|
|
7818
9782
|
);
|
|
7819
9783
|
}
|
|
@@ -7833,44 +9797,97 @@ export class AgentFramework {
|
|
|
7833
9797
|
for (let i = 0; i < 22; i++) suffix += alphabet[Math.floor(Math.random() * alphabet.length)];
|
|
7834
9798
|
const toolUseId = `toolu_01${suffix}`;
|
|
7835
9799
|
|
|
7836
|
-
|
|
7837
|
-
|
|
7838
|
-
|
|
7839
|
-
|
|
7840
|
-
|
|
7841
|
-
|
|
7842
|
-
|
|
7843
|
-
|
|
7844
|
-
|
|
7845
|
-
//
|
|
7846
|
-
//
|
|
7847
|
-
|
|
7848
|
-
|
|
7849
|
-
|
|
7850
|
-
|
|
7851
|
-
|
|
7852
|
-
|
|
7853
|
-
|
|
7854
|
-
|
|
7855
|
-
|
|
7856
|
-
|
|
7857
|
-
|
|
7858
|
-
|
|
7859
|
-
|
|
9800
|
+
// Reserve the agent against turn start for the WHOLE operation, before
|
|
9801
|
+
// the first await (#145). The idle check above is a point in time; the
|
|
9802
|
+
// tool execution and the result build both await, and a wake arriving
|
|
9803
|
+
// in either gap used to start a real turn — then the pair below wrote
|
|
9804
|
+
// straight through the context manager into a turn that never saw it,
|
|
9805
|
+
// the exact wire-order corruption this method exists to avoid. Holding
|
|
9806
|
+
// a turn token is the same invariant the scheduler respects (turn-alive
|
|
9807
|
+
// requeues wakes) and the addMessage guard reads (cross-turn writers
|
|
9808
|
+
// defer), so for the duration this behaves like a turn with no stream.
|
|
9809
|
+
// Token-matched release in finally, same leak-proofing as
|
|
9810
|
+
// startAgentStream: a token nobody clears wedges the agent.
|
|
9811
|
+
const turnToken = this.nextTurnToken++;
|
|
9812
|
+
this.activeTurnTokens.set(agentName, turnToken);
|
|
9813
|
+
let ownedToEnd = false;
|
|
9814
|
+
try {
|
|
9815
|
+
const started = Date.now();
|
|
9816
|
+
const result = await this.executeToolCall({
|
|
9817
|
+
id: toolUseId,
|
|
9818
|
+
name: toolName,
|
|
9819
|
+
input,
|
|
9820
|
+
callerAgentName: agentName,
|
|
9821
|
+
});
|
|
9822
|
+
const durationMs = Date.now() - started;
|
|
9823
|
+
|
|
9824
|
+
// Store the pair through the same shapes the ordinary path uses. Build
|
|
9825
|
+
// the result blocks BEFORE storing the tool_use: the spill path awaits.
|
|
9826
|
+
// A message arriving during either await is deferred by the turn
|
|
9827
|
+
// token (as during a real tool call) and lands AFTER the pair, never
|
|
9828
|
+
// between tool_use and its tool_result. The two addMessage calls
|
|
9829
|
+
// below are synchronous and adjacent — nothing can interleave.
|
|
9830
|
+
const { blocks } = await this.buildStoredToolResultContent(
|
|
9831
|
+
agentName,
|
|
9832
|
+
[{ id: toolUseId, name: toolName, input, result, durationMs }],
|
|
9833
|
+
this.resolveToolResultInlineCap(agent).cap,
|
|
9834
|
+
);
|
|
9835
|
+
const toolUse: ContentBlock[] = [
|
|
9836
|
+
{ type: 'tool_use', id: toolUseId, name: toolName, input } as ContentBlock,
|
|
9837
|
+
];
|
|
9838
|
+
// Defensive: no path replaces a live token any more (the provider-
|
|
9839
|
+
// admission re-entry re-tests turn-alive and requeues instead), but
|
|
9840
|
+
// if one ever does, the side effect has happened and the pair must
|
|
9841
|
+
// still be stored — not under that turn: queue it as a unit for the
|
|
9842
|
+
// next flush (drainDeferredFor keeps order; the tool_result entry is
|
|
9843
|
+
// never split from its tool_use because both ride the same drain).
|
|
9844
|
+
// The finally below flushes it itself if that turn is already over.
|
|
9845
|
+
ownedToEnd = this.activeTurnTokens.get(agentName) === turnToken;
|
|
9846
|
+
if (ownedToEnd) {
|
|
9847
|
+
const cm = agent.getContextManager();
|
|
9848
|
+
cm.addMessage(agentName, toolUse);
|
|
9849
|
+
cm.addMessage('user', blocks);
|
|
9850
|
+
} else {
|
|
9851
|
+
this.deferredMessages.push(
|
|
9852
|
+
{ id: randomUUID(), seq: ++this.deferredSeq, participant: agentName, content: toolUse, forAgent: agentName },
|
|
9853
|
+
{ id: randomUUID(), seq: ++this.deferredSeq, participant: 'user', content: blocks, forAgent: agentName },
|
|
9854
|
+
);
|
|
9855
|
+
}
|
|
7860
9856
|
|
|
7861
|
-
|
|
7862
|
-
|
|
7863
|
-
|
|
7864
|
-
|
|
7865
|
-
|
|
7866
|
-
|
|
7867
|
-
|
|
7868
|
-
|
|
7869
|
-
|
|
7870
|
-
|
|
7871
|
-
|
|
7872
|
-
|
|
7873
|
-
|
|
9857
|
+
this.emitTrace({
|
|
9858
|
+
type: 'puppet:tool-call',
|
|
9859
|
+
agentName,
|
|
9860
|
+
toolName,
|
|
9861
|
+
toolUseId,
|
|
9862
|
+
isError: !!result.isError,
|
|
9863
|
+
durationMs,
|
|
9864
|
+
...(ownedToEnd ? {} : { deferred: true }),
|
|
9865
|
+
});
|
|
9866
|
+
console.log(
|
|
9867
|
+
`[puppet] ${agentName}: ${toolName} → ${result.isError ? 'ERROR' : 'ok'} ` +
|
|
9868
|
+
`in ${durationMs}ms (${toolUseId})${ownedToEnd ? '' : ' — deferred behind a live turn'}`,
|
|
9869
|
+
);
|
|
9870
|
+
return { toolUseId, result };
|
|
9871
|
+
} finally {
|
|
9872
|
+
if (this.activeTurnTokens.get(agentName) === turnToken) this.activeTurnTokens.delete(agentName);
|
|
9873
|
+
// Messages deferred while we held the token land now, after the pair
|
|
9874
|
+
// — the same end-of-turn flush driveStream performs, under the same
|
|
9875
|
+
// tool-cycle guard. (No wake is requested: the agent sees the pair,
|
|
9876
|
+
// and anything that arrived meanwhile, on its next turn.) Keyed on
|
|
9877
|
+
// "no turn alive", not on token ownership: if a turn did replace us
|
|
9878
|
+
// and has already ended, its flush ran before our pair was queued —
|
|
9879
|
+
// nobody else will flush it, so we do.
|
|
9880
|
+
if (!this.activeTurnTokens.has(agentName)
|
|
9881
|
+
&& this.deferredMessages.length > 0 && this.pendingAssistantBlocks.size === 0) {
|
|
9882
|
+
for (const msg of this.drainDeferredFor(agentName)) {
|
|
9883
|
+
this.addMessage(msg.participant, msg.content, msg.metadata, {
|
|
9884
|
+
deferredWriteId: msg.id,
|
|
9885
|
+
...(msg.forAgent ? { forAgent: msg.forAgent } : {}),
|
|
9886
|
+
});
|
|
9887
|
+
}
|
|
9888
|
+
this.ackDeferredWrites();
|
|
9889
|
+
}
|
|
9890
|
+
}
|
|
7874
9891
|
}
|
|
7875
9892
|
|
|
7876
9893
|
private async executeToolCallFrom(call: ToolCall, origin: ChannelToolOrigin): Promise<ToolResult> {
|
|
@@ -8417,10 +10434,16 @@ export class AgentFramework {
|
|
|
8417
10434
|
try {
|
|
8418
10435
|
const result = await workspace.writeBinary(path, Buffer.from(content, 'utf8'), 'text/plain');
|
|
8419
10436
|
if (result.success) {
|
|
10437
|
+
// JSON serialization can expand each read-back code unit to six
|
|
10438
|
+
// chars (\\u0000). Leave room for the page's metadata and a possible
|
|
10439
|
+
// extra code unit at a surrogate-pair boundary.
|
|
10440
|
+
const pageOverhead = JSON.stringify(path).length + 256;
|
|
10441
|
+
const readLimit = Math.max(1, Math.min(2000, Math.floor((cap - pageOverhead) / 6) - 1));
|
|
8420
10442
|
return {
|
|
8421
10443
|
text: head
|
|
8422
|
-
+ `\n\n[truncated — showing ${head.length} of ${content.length} chars; full content: workspace file ${path}
|
|
8423
|
-
+
|
|
10444
|
+
+ `\n\n[truncated — showing ${head.length} of ${content.length} chars; full content: workspace file ${path}`
|
|
10445
|
+
+ `. Page with workspace--read ${JSON.stringify({ path, offsetChars: 0, limitChars: readLimit })}; `
|
|
10446
|
+
+ 'use nextOffsetChars as offsetChars until null, keeping limitChars. Or raise the inline cap via '
|
|
8424
10447
|
+ 'agent_settings update tool_result_inline_max_chars.]'
|
|
8425
10448
|
+ kept,
|
|
8426
10449
|
filePath: path,
|
|
@@ -8773,11 +10796,18 @@ export class AgentFramework {
|
|
|
8773
10796
|
// Blob threshold: 10KB - typical context-heavy requests exceed this
|
|
8774
10797
|
const BLOB_THRESHOLD = 10000;
|
|
8775
10798
|
|
|
10799
|
+
// Both branches persist the JSON view, never the live object: a compiled
|
|
10800
|
+
// request carries non-JSON members (the kv-unified `onCacheWireReceipt`
|
|
10801
|
+
// hook is a function) and Chronicle's JSON bridge rejects those with
|
|
10802
|
+
// "JS functions cannot be represented as a serde_json::Value" — which,
|
|
10803
|
+
// thrown from the failure path, masked the failure being logged.
|
|
8776
10804
|
if (entry.request && typeof entry.request === 'object') {
|
|
8777
10805
|
const requestJson = JSON.stringify(entry.request);
|
|
8778
10806
|
if (requestJson.length > BLOB_THRESHOLD) {
|
|
8779
10807
|
const blobId = this.store.storeBlob(Buffer.from(requestJson), 'application/json');
|
|
8780
10808
|
entryToStore.request = { blobId };
|
|
10809
|
+
} else {
|
|
10810
|
+
entryToStore.request = JSON.parse(requestJson);
|
|
8781
10811
|
}
|
|
8782
10812
|
}
|
|
8783
10813
|
|
|
@@ -8786,6 +10816,8 @@ export class AgentFramework {
|
|
|
8786
10816
|
if (responseJson.length > BLOB_THRESHOLD) {
|
|
8787
10817
|
const blobId = this.store.storeBlob(Buffer.from(responseJson), 'application/json');
|
|
8788
10818
|
entryToStore.response = { blobId };
|
|
10819
|
+
} else {
|
|
10820
|
+
entryToStore.response = JSON.parse(responseJson);
|
|
8789
10821
|
}
|
|
8790
10822
|
}
|
|
8791
10823
|
|
|
@@ -9057,6 +11089,14 @@ export class AgentFramework {
|
|
|
9057
11089
|
* this is reachable from timer callbacks that may outlive an agent).
|
|
9058
11090
|
*/
|
|
9059
11091
|
forAgent?: string;
|
|
11092
|
+
/**
|
|
11093
|
+
* Set when the message is being flushed FROM the durable deferred
|
|
11094
|
+
* queue: it is the entry's one durable identity. A write stamps it
|
|
11095
|
+
* into `metadata.deferredWriteId` (boot dedup); a re-deferral moves
|
|
11096
|
+
* the same logical entry back to pending under the same id instead of
|
|
11097
|
+
* minting a second, independently replayable one.
|
|
11098
|
+
*/
|
|
11099
|
+
deferredWriteId?: string;
|
|
9060
11100
|
}
|
|
9061
11101
|
): MessageId {
|
|
9062
11102
|
// Route to the named agent, else the primary (not ephemeral subagents).
|
|
@@ -9092,6 +11132,10 @@ export class AgentFramework {
|
|
|
9092
11132
|
// their own wake), at the agent's next tool boundary (where they are
|
|
9093
11133
|
// ALSO injected into the live stream — hear-while-acting), or in
|
|
9094
11134
|
// driveStream's finally when the turn ends.
|
|
11135
|
+
// Quiesce (issue #122): the window exists to run compression/refold
|
|
11136
|
+
// against a FROZEN context — module events and api message.send must not
|
|
11137
|
+
// append mid-surgery. Deferred here, flushed by resume(). tool_result
|
|
11138
|
+
// still lands (a draining turn's continuation depends on it).
|
|
9095
11139
|
const hasToolResult = content.some(b => b.type === 'tool_result');
|
|
9096
11140
|
// Explicitly-targeted deliveries scope the tool-cycle check to the
|
|
9097
11141
|
// target (pendingAssistantBlocks is keyed by agent); the default path
|
|
@@ -9102,15 +11146,30 @@ export class AgentFramework {
|
|
|
9102
11146
|
: this.pendingAssistantBlocks.size > 0;
|
|
9103
11147
|
if (
|
|
9104
11148
|
!hasToolResult &&
|
|
9105
|
-
(
|
|
11149
|
+
(this.quiesced ||
|
|
11150
|
+
midToolCycle ||
|
|
9106
11151
|
this.activeTurnTokens.has(agent.name) ||
|
|
9107
11152
|
this.activeStreams.has(agent.name))
|
|
9108
11153
|
) {
|
|
9109
|
-
|
|
11154
|
+
// A re-deferral of a drained entry keeps its id and leaves the un-acked
|
|
11155
|
+
// set: one logical message, one durable identity — never two entries.
|
|
11156
|
+
const id = opts?.deferredWriteId ?? randomUUID();
|
|
11157
|
+
let seq: number | undefined;
|
|
11158
|
+
if (opts?.deferredWriteId) {
|
|
11159
|
+
const prior = this.unackedDeferredWrites.find((m) => m.id === id);
|
|
11160
|
+
seq = prior?.seq;
|
|
11161
|
+
this.unackedDeferredWrites = this.unackedDeferredWrites.filter((m) => m.id !== id);
|
|
11162
|
+
}
|
|
11163
|
+
this.deferredMessages.push({ id, seq: seq ?? ++this.deferredSeq, participant, content, metadata, forAgent: opts?.forAgent });
|
|
11164
|
+
if (this.quiesced || this.deferredWritesPersisted) this.persistDeferredWrites();
|
|
9110
11165
|
return '' as MessageId; // Deferred — flushed at the target's next boundary
|
|
9111
11166
|
}
|
|
9112
11167
|
|
|
9113
|
-
return agent.getContextManager().addMessage(
|
|
11168
|
+
return agent.getContextManager().addMessage(
|
|
11169
|
+
participant,
|
|
11170
|
+
content,
|
|
11171
|
+
opts?.deferredWriteId ? withDeferredWriteId(metadata, opts.deferredWriteId) : metadata,
|
|
11172
|
+
);
|
|
9114
11173
|
}
|
|
9115
11174
|
|
|
9116
11175
|
/**
|
|
@@ -9119,6 +11178,9 @@ export class AgentFramework {
|
|
|
9119
11178
|
* targets' messages queued for their own boundaries.
|
|
9120
11179
|
*/
|
|
9121
11180
|
private drainDeferredFor(agentName: string): Array<{
|
|
11181
|
+
/** Durable identity — every write of this entry must carry it
|
|
11182
|
+
* (`withDeferredWriteId` / `opts.deferredWriteId`). */
|
|
11183
|
+
id: string;
|
|
9122
11184
|
participant: string;
|
|
9123
11185
|
content: ContentBlock[];
|
|
9124
11186
|
metadata?: MessageMetadata;
|
|
@@ -9132,6 +11194,12 @@ export class AgentFramework {
|
|
|
9132
11194
|
(target === agentName ? mine : rest).push(msg);
|
|
9133
11195
|
}
|
|
9134
11196
|
this.deferredMessages = rest;
|
|
11197
|
+
mine.sort(bySeq);
|
|
11198
|
+
// The drained messages are about to be written by the caller; they stay
|
|
11199
|
+
// in the DURABLE queue (as un-acked) until the caller's
|
|
11200
|
+
// ackDeferredWrites() has synced the chronicle. Removing them here would
|
|
11201
|
+
// acknowledge before the write is durable.
|
|
11202
|
+
this.handOffDeferredWrites(mine);
|
|
9135
11203
|
return mine;
|
|
9136
11204
|
}
|
|
9137
11205
|
|
|
@@ -10123,8 +12191,21 @@ export class AgentFramework {
|
|
|
10123
12191
|
for (const connection of this.mcplServerRegistry?.getAllServers() ?? []) {
|
|
10124
12192
|
// ready() can synchronously flush a nested list-change notification that
|
|
10125
12193
|
// installs a newer global generation. Never let this older completion
|
|
10126
|
-
// release any remaining server behind that newer gate.
|
|
12194
|
+
// release any remaining server behind that newer gate. (The quiesce
|
|
12195
|
+
// check sits AFTER this guard: readyControlPlane can also synchronously
|
|
12196
|
+
// flush a control event that installs a newer barrier.)
|
|
10127
12197
|
if (this.discordAwarenessBarrier !== null) return false;
|
|
12198
|
+
// Host quiesce (issue #122): this is the single point every barrier
|
|
12199
|
+
// completion and reconnect flow funnels through, so holding here keeps
|
|
12200
|
+
// data planes paused across awareness drains and reconnects for the
|
|
12201
|
+
// whole window. Control planes come up normally (host/command rides the
|
|
12202
|
+
// control plane, so resume stays deliverable); resume() performs the
|
|
12203
|
+
// real ready() flush through this same funnel once its feasibility gate
|
|
12204
|
+
// passes and the flag is cleared.
|
|
12205
|
+
if (this.quiesced) {
|
|
12206
|
+
connection.readyControlPlane();
|
|
12207
|
+
continue;
|
|
12208
|
+
}
|
|
10128
12209
|
connection.ready();
|
|
10129
12210
|
}
|
|
10130
12211
|
return this.discordAwarenessBarrier === null;
|