@animalabs/agent-framework 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -1
- package/dist/src/agent.d.ts +32 -0
- package/dist/src/agent.d.ts.map +1 -1
- package/dist/src/agent.js +55 -0
- package/dist/src/agent.js.map +1 -1
- package/dist/src/api/server.d.ts +9 -0
- package/dist/src/api/server.d.ts.map +1 -1
- package/dist/src/api/server.js +156 -0
- package/dist/src/api/server.js.map +1 -1
- package/dist/src/api/types.d.ts +15 -0
- package/dist/src/api/types.d.ts.map +1 -1
- package/dist/src/framework.d.ts +423 -3
- package/dist/src/framework.d.ts.map +1 -1
- package/dist/src/framework.js +1857 -75
- package/dist/src/framework.js.map +1 -1
- package/dist/src/gate/event-gate.d.ts +16 -0
- package/dist/src/gate/event-gate.d.ts.map +1 -1
- package/dist/src/gate/event-gate.js +41 -3
- package/dist/src/gate/event-gate.js.map +1 -1
- package/dist/src/index.d.ts +4 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +3 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/mcpl/channel-registry.d.ts +172 -0
- package/dist/src/mcpl/channel-registry.d.ts.map +1 -1
- package/dist/src/mcpl/channel-registry.js +444 -2
- package/dist/src/mcpl/channel-registry.js.map +1 -1
- package/dist/src/mcpl/server-connection.d.ts +9 -0
- package/dist/src/mcpl/server-connection.d.ts.map +1 -1
- package/dist/src/mcpl/server-connection.js +71 -13
- package/dist/src/mcpl/server-connection.js.map +1 -1
- package/dist/src/modules/history/index.d.ts +89 -13
- package/dist/src/modules/history/index.d.ts.map +1 -1
- package/dist/src/modules/history/index.js +655 -24
- package/dist/src/modules/history/index.js.map +1 -1
- package/dist/src/operator-log.d.ts +78 -0
- package/dist/src/operator-log.d.ts.map +1 -0
- package/dist/src/operator-log.js +102 -0
- package/dist/src/operator-log.js.map +1 -0
- package/dist/src/recovery/discord-awareness-outbox.d.ts +10 -0
- package/dist/src/recovery/discord-awareness-outbox.d.ts.map +1 -1
- package/dist/src/recovery/discord-awareness-outbox.js +20 -0
- package/dist/src/recovery/discord-awareness-outbox.js.map +1 -1
- package/dist/src/types/framework.d.ts +56 -0
- package/dist/src/types/framework.d.ts.map +1 -1
- package/dist/src/types/trace.d.ts +36 -0
- package/dist/src/types/trace.d.ts.map +1 -1
- package/dist/test/agent-runtime-settings.test.js +199 -1
- package/dist/test/agent-runtime-settings.test.js.map +1 -1
- package/dist/test/api-admin-gate.test.d.ts +2 -0
- package/dist/test/api-admin-gate.test.d.ts.map +1 -0
- package/dist/test/api-admin-gate.test.js +116 -0
- package/dist/test/api-admin-gate.test.js.map +1 -0
- package/dist/test/channel-label-history.test.d.ts +2 -0
- package/dist/test/channel-label-history.test.d.ts.map +1 -0
- package/dist/test/channel-label-history.test.js +424 -0
- package/dist/test/channel-label-history.test.js.map +1 -0
- package/dist/test/discord-awareness-outbox.test.js +16 -0
- package/dist/test/discord-awareness-outbox.test.js.map +1 -1
- package/dist/test/event-gate.test.js +44 -0
- package/dist/test/event-gate.test.js.map +1 -1
- package/dist/test/helpers/quiesce-crash-child.d.ts +2 -0
- package/dist/test/helpers/quiesce-crash-child.d.ts.map +1 -0
- package/dist/test/helpers/quiesce-crash-child.js +156 -0
- package/dist/test/helpers/quiesce-crash-child.js.map +1 -0
- package/dist/test/history-module-overview.test.d.ts +2 -0
- package/dist/test/history-module-overview.test.d.ts.map +1 -0
- package/dist/test/history-module-overview.test.js +711 -0
- package/dist/test/history-module-overview.test.js.map +1 -0
- package/dist/test/history-module.test.js +146 -0
- package/dist/test/history-module.test.js.map +1 -1
- package/dist/test/host-quiesce.test.d.ts +2 -0
- package/dist/test/host-quiesce.test.d.ts.map +1 -0
- package/dist/test/host-quiesce.test.js +863 -0
- package/dist/test/host-quiesce.test.js.map +1 -0
- package/dist/test/liveness-watchdog.test.js +18 -2
- package/dist/test/liveness-watchdog.test.js.map +1 -1
- package/dist/test/operator-surgery.test.d.ts +2 -0
- package/dist/test/operator-surgery.test.d.ts.map +1 -0
- package/dist/test/operator-surgery.test.js +398 -0
- package/dist/test/operator-surgery.test.js.map +1 -0
- package/dist/test/provider-acceleration-cooldown.test.js +145 -0
- package/dist/test/provider-acceleration-cooldown.test.js.map +1 -1
- package/dist/test/puppet-tool-call.test.js +5 -0
- package/dist/test/puppet-tool-call.test.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
- package/src/agent.ts +69 -0
- package/src/api/server.ts +159 -2
- package/src/api/types.ts +15 -0
- package/src/framework.ts +2155 -96
- package/src/gate/event-gate.ts +47 -4
- package/src/index.ts +5 -1
- package/src/mcpl/channel-registry.ts +495 -2
- package/src/mcpl/server-connection.ts +73 -13
- package/src/modules/history/index.ts +689 -25
- package/src/operator-log.ts +132 -0
- package/src/recovery/discord-awareness-outbox.ts +19 -0
- package/src/types/framework.ts +63 -0
- package/src/types/trace.ts +42 -0
package/dist/src/framework.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { join } from 'node:path';
|
|
1
|
+
import { dirname, join } from 'node:path';
|
|
2
2
|
import { INLINE_WITHHELD_TEXT, classifyBlock, isInlineContradiction, referenceRegistry, referenceStubOrNull } from './mcpl/references.js';
|
|
3
3
|
import { ReferenceFetcher, DEFAULT_FETCH_MAX_BYTES, EAGER_FETCH_TIMEOUT_MS } from './mcpl/reference-fetcher.js';
|
|
4
|
-
import { appendFileSync, mkdirSync } from 'node:fs';
|
|
4
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
|
5
5
|
import { JsStore } from '@animalabs/chronicle';
|
|
6
6
|
import { MembraneError } from '@animalabs/membrane';
|
|
7
7
|
import { ContextManager, PassthroughStrategy, WindowedPassthroughStrategy } from '@animalabs/context-manager';
|
|
@@ -284,7 +284,23 @@ import { EventGate, formatShadowWarning } from './gate/event-gate.js';
|
|
|
284
284
|
import { UsageTracker } from './usage/usage-tracker.js';
|
|
285
285
|
import { formatZonedDateTime, resolveTimeZone } from './timezone.js';
|
|
286
286
|
import { DEFAULT_DISCORD_AWARENESS_EMOJI, DiscordAwarenessOutbox, defaultDiscordAwarenessOutboxPath, extractDiscordAwarenessRefs, } from './recovery/discord-awareness-outbox.js';
|
|
287
|
+
import { OperatorActionError, OperatorLog, capIds, defaultOperatorLogPath, } from './operator-log.js';
|
|
287
288
|
const FRAMEWORK_STATE_ID = 'framework/state';
|
|
289
|
+
/** Snapshot slot mirroring context writes deferred while quiesced (#122). */
|
|
290
|
+
const DEFERRED_WRITES_ID = 'framework/deferred-writes';
|
|
291
|
+
/** Above this serialized size the deferred-write mirror is skipped (kept in
|
|
292
|
+
* memory only) rather than rewriting a multi-MB snapshot per deferral. */
|
|
293
|
+
const DEFERRED_WRITES_PERSIST_CAP_BYTES = 4 * 1024 * 1024;
|
|
294
|
+
const bySeq = (a, b) => a.seq - b.seq;
|
|
295
|
+
/** Stamp a deferred write's durable id into the metadata it is stored with
|
|
296
|
+
* (boot recovery dedups replays by it). Idempotent for an already-stamped
|
|
297
|
+
* message. */
|
|
298
|
+
function withDeferredWriteId(metadata, id) {
|
|
299
|
+
return { ...(metadata ?? {}), deferredWriteId: id };
|
|
300
|
+
}
|
|
301
|
+
function isTurnContinuation(reason) {
|
|
302
|
+
return reason === 'context_budget_restart' || reason === 'tool_results_ready';
|
|
303
|
+
}
|
|
288
304
|
const CONVERSATION_ROUTER_STATE_ID = 'framework/conversation-router';
|
|
289
305
|
const INFERENCE_LOG_ID = 'framework/inference-log';
|
|
290
306
|
const PROCESS_LOG_ID = 'framework/process-log';
|
|
@@ -308,6 +324,37 @@ class DiscordAwarenessAccountingError extends Error {
|
|
|
308
324
|
this.name = 'DiscordAwarenessAccountingError';
|
|
309
325
|
}
|
|
310
326
|
}
|
|
327
|
+
/**
|
|
328
|
+
* Thrown by `updateAgentRuntimeSettings` when an immediate budget change would
|
|
329
|
+
* put the agent in an un-compilable layout: the folded floor renders more
|
|
330
|
+
* tokens than the hard budget admits, so every subsequent compile would
|
|
331
|
+
* OverBudget-wedge the agent (2026-08-21: mythos, 550k→260k set live against a
|
|
332
|
+
* ~450k floor — hours of hard-down). Overridable with `{ allowInfeasible: true }`.
|
|
333
|
+
* Paced descents never throw this: a non-immediate decrease leaves the compile
|
|
334
|
+
* budget untouched and converges only as far as the floor allows.
|
|
335
|
+
*/
|
|
336
|
+
export class BudgetPreflightError extends Error {
|
|
337
|
+
preview;
|
|
338
|
+
constructor(message, preview) {
|
|
339
|
+
super(message);
|
|
340
|
+
this.preview = preview;
|
|
341
|
+
this.name = 'BudgetPreflightError';
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
/**
|
|
345
|
+
* Thrown by `resume()` when the current runtime settings do not compile for
|
|
346
|
+
* one or more agents — returning to service would OverBudget-wedge them on
|
|
347
|
+
* the first wake. Drain quarantine / advance merges to lower the floor, or
|
|
348
|
+
* pass `{ force: true }` after deciding the verdicts are acceptable.
|
|
349
|
+
*/
|
|
350
|
+
export class ResumeBlockedError extends Error {
|
|
351
|
+
verdicts;
|
|
352
|
+
constructor(message, verdicts) {
|
|
353
|
+
super(message);
|
|
354
|
+
this.verdicts = verdicts;
|
|
355
|
+
this.name = 'ResumeBlockedError';
|
|
356
|
+
}
|
|
357
|
+
}
|
|
311
358
|
/**
|
|
312
359
|
* Default inference policy - infer if any request exists for the agent.
|
|
313
360
|
*/
|
|
@@ -316,6 +363,64 @@ class DefaultInferencePolicy {
|
|
|
316
363
|
return requests.some((r) => r.agentName === agentName);
|
|
317
364
|
}
|
|
318
365
|
}
|
|
366
|
+
const SURGERY_SCAN_WINDOW = 500;
|
|
367
|
+
/**
|
|
368
|
+
* Slot index of `messageId` on the active branch, or -1. Scans from the tail
|
|
369
|
+
* in blob-free windows: operator targets are almost always recent, and a
|
|
370
|
+
* full `getAllMessages()` re-inflates every attachment on the branch.
|
|
371
|
+
*/
|
|
372
|
+
function locateMessageIndex(cm, messageId) {
|
|
373
|
+
const hit = locateMessageIndices(cm, new Set([messageId])).get(messageId);
|
|
374
|
+
return hit ? hit.index : -1;
|
|
375
|
+
}
|
|
376
|
+
/** Tail-first scan resolving many ids at once; stops as soon as all are found. */
|
|
377
|
+
function locateMessageIndices(cm, ids) {
|
|
378
|
+
const found = new Map();
|
|
379
|
+
const total = cm.getMessageCount();
|
|
380
|
+
for (let end = total; end > 0 && found.size < ids.size; end -= SURGERY_SCAN_WINDOW) {
|
|
381
|
+
const start = Math.max(0, end - SURGERY_SCAN_WINDOW);
|
|
382
|
+
const { messages, startIndex } = cm.getMessageWindow(start, end - start, { resolveBlobs: false });
|
|
383
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
384
|
+
const id = String(messages[i].id);
|
|
385
|
+
if (ids.has(id) && !found.has(id)) {
|
|
386
|
+
found.set(id, {
|
|
387
|
+
index: startIndex + i,
|
|
388
|
+
...(messages[i].bodyGroupId ? { bodyGroupId: messages[i].bodyGroupId } : {}),
|
|
389
|
+
});
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
return found;
|
|
394
|
+
}
|
|
395
|
+
/**
|
|
396
|
+
* Inclusive slot range of the body-group run containing `index` (the slot
|
|
397
|
+
* itself when it is not sharded). Delegates to the store's own
|
|
398
|
+
* `alignToBodyGroups` edge walk — one contiguity assumption, not two.
|
|
399
|
+
*/
|
|
400
|
+
function bodyGroupRun(cm, index) {
|
|
401
|
+
const win = cm.getMessageWindow(index, 1, { alignToBodyGroups: true, resolveBlobs: false });
|
|
402
|
+
const from = win.startIndex;
|
|
403
|
+
const to = win.startIndex + win.messages.length - 1;
|
|
404
|
+
return {
|
|
405
|
+
from,
|
|
406
|
+
to,
|
|
407
|
+
fromId: String(win.messages[0].id),
|
|
408
|
+
toId: String(win.messages[win.messages.length - 1].id),
|
|
409
|
+
group: win.messages[0].bodyGroupId !== undefined,
|
|
410
|
+
};
|
|
411
|
+
}
|
|
412
|
+
function describeRequester(r) {
|
|
413
|
+
if (!r)
|
|
414
|
+
return 'unknown';
|
|
415
|
+
return `${r.name ?? r.id ?? 'unknown'} (${r.via})`;
|
|
416
|
+
}
|
|
417
|
+
function hostCommandRequester(serverId, params) {
|
|
418
|
+
return {
|
|
419
|
+
via: `host-command:${serverId}`,
|
|
420
|
+
...(params.requesterName ? { name: params.requesterName } : {}),
|
|
421
|
+
...(params.requesterId ? { id: params.requesterId } : {}),
|
|
422
|
+
};
|
|
423
|
+
}
|
|
319
424
|
function isPermanentDiscordReactionFailure(message) {
|
|
320
425
|
return /unknown message|unknown channel|missing access|missing permissions|missing permission|cannot access|channel .* not found|message .* not found/i
|
|
321
426
|
.test(message);
|
|
@@ -421,6 +526,17 @@ export class AgentFramework {
|
|
|
421
526
|
activeTriggerChannels = new Map();
|
|
422
527
|
running = false;
|
|
423
528
|
loopPromise = null;
|
|
529
|
+
/** Quiesce/maintenance mode (issue #122). DELIBERATELY separate from
|
|
530
|
+
* `running`, which is overloaded as the runLoop condition, the maintenance
|
|
531
|
+
* admission guard, AND the per-tick loop condition — expressing "paused" by
|
|
532
|
+
* clearing `running` would kill the very maintenance machinery quiesce
|
|
533
|
+
* exists to keep hot, plus store sync and tool-result processing. While
|
|
534
|
+
* quiesced: runLoop, syncTimer, maintenanceTimer, and the watchdog all keep
|
|
535
|
+
* running; only new turns (wake gate in processInferenceRequests) and MCPL
|
|
536
|
+
* data planes are held. */
|
|
537
|
+
quiesced = false;
|
|
538
|
+
quiesceReason;
|
|
539
|
+
quiescedAt;
|
|
424
540
|
traceListeners = [];
|
|
425
541
|
syncIntervalMs;
|
|
426
542
|
syncTimer = null;
|
|
@@ -436,6 +552,7 @@ export class AgentFramework {
|
|
|
436
552
|
providerAccelerationLastRecovery = new Map();
|
|
437
553
|
providerAccelerationDefaultCooldownMs = PROVIDER_ACCELERATION_DEFAULT_COOLDOWN_MS;
|
|
438
554
|
providerAccelerationJitterMs = PROVIDER_ACCELERATION_JITTER_MS;
|
|
555
|
+
providerHoldHook;
|
|
439
556
|
providerAdmissionClosed = false;
|
|
440
557
|
/** Last time we reported stale (busy-requeued) inference requests, per agent. */
|
|
441
558
|
staleWarnAt = new Map();
|
|
@@ -591,6 +708,36 @@ export class AgentFramework {
|
|
|
591
708
|
subconsciousConfig = null;
|
|
592
709
|
/** Tune-out coordinator (issue #77); non-null iff subconscious + channels. */
|
|
593
710
|
tuneOutCoordinator = null;
|
|
711
|
+
/** Agents an abandon could not cancel (token held, no stream) — surfaced
|
|
712
|
+
* in HostModeStatus until the next quiesce/resume. */
|
|
713
|
+
lastUnabandonable = [];
|
|
714
|
+
/** True while the deferred-write queue has a persisted mirror (quiesced). */
|
|
715
|
+
deferredWritesPersisted = false;
|
|
716
|
+
deferredWritesCapWarned = false;
|
|
717
|
+
/** Branch-independent recovery files (see FrameworkConfig.hostModePath /
|
|
718
|
+
* deferredWritesPath). Undefined → branch-local slot fallback. */
|
|
719
|
+
hostModePath;
|
|
720
|
+
deferredWritesPath;
|
|
721
|
+
recoveryFallbackWarned = false;
|
|
722
|
+
/**
|
|
723
|
+
* Deferred writes that have been handed to a context manager but whose
|
|
724
|
+
* chronicle state is NOT yet durably synced. They stay in the durable
|
|
725
|
+
* recovery queue until `ackDeferredWrites()` syncs the store — chronicle
|
|
726
|
+
* persists the slot-chain head only on sync(), so acknowledging on append
|
|
727
|
+
* alone would let a hard exit forget an accepted message.
|
|
728
|
+
*/
|
|
729
|
+
unackedDeferredWrites = [];
|
|
730
|
+
/** Monotonic order stamp for deferred writes: the durable queue is always
|
|
731
|
+
* written, restored, drained and flushed in `seq` order, whatever the
|
|
732
|
+
* pending/un-acked split — a re-deferred entry keeps its place. */
|
|
733
|
+
deferredSeq = 0;
|
|
734
|
+
/**
|
|
735
|
+
* Per target agent: the store's message count when the OLDEST currently
|
|
736
|
+
* un-acked entry for it was handed off. Persisted with the queue so a boot
|
|
737
|
+
* after an interrupted flush knows exactly which slot range the batch
|
|
738
|
+
* could occupy and dedups against all of it — never a fixed tail.
|
|
739
|
+
*/
|
|
740
|
+
deferredScanFrom = new Map();
|
|
594
741
|
deferredMessages = [];
|
|
595
742
|
// Turn-alive markers (2026-07-31 Mythos phantom-skip incident): agentName →
|
|
596
743
|
// token of the turn currently in progress. Set at startAgentStream ENTRY —
|
|
@@ -702,6 +849,16 @@ export class AgentFramework {
|
|
|
702
849
|
discordAwarenessOutbox = null;
|
|
703
850
|
discordAwarenessEmoji = DEFAULT_DISCORD_AWARENESS_EMOJI;
|
|
704
851
|
discordAwarenessDeadlineMs = DEFAULT_DISCORD_AWARENESS_DEADLINE_MS;
|
|
852
|
+
/** Durable JSONL record of operator-initiated mutations (see operator-log.ts). */
|
|
853
|
+
operatorLog;
|
|
854
|
+
/**
|
|
855
|
+
* Store-wide admission barrier while a live surgery (rollback/suppress)
|
|
856
|
+
* holds the store. Per-agent turn tokens cover the agents registered when
|
|
857
|
+
* the reservation was taken; this flag covers everyone else — an ephemeral
|
|
858
|
+
* agent admitted mid-switch, a wake for an agent created later — at the
|
|
859
|
+
* scheduler, ephemeral admission and puppet entry points.
|
|
860
|
+
*/
|
|
861
|
+
surgeryHold = null;
|
|
705
862
|
/** Serialize per-server drains so reconnect and an online undo cannot race. */
|
|
706
863
|
discordAwarenessDrains = new Map();
|
|
707
864
|
/** Framework-global inference gate; older generations cannot release it. */
|
|
@@ -713,7 +870,8 @@ export class AgentFramework {
|
|
|
713
870
|
usageTracker;
|
|
714
871
|
/** Presentation-only wall-clock zone; persistence remains UTC/epoch. */
|
|
715
872
|
timeZone;
|
|
716
|
-
constructor(store, ownsStore, membrane, inferencePolicy, errorPolicy, syncIntervalMs, maintenanceIntervalMs, processLoggingPersist, processLoggingBroadcast, timeZone, discordAwarenessOutbox, discordAwarenessEmoji, discordAwarenessDeadlineMs) {
|
|
873
|
+
constructor(store, ownsStore, membrane, inferencePolicy, errorPolicy, syncIntervalMs, maintenanceIntervalMs, processLoggingPersist, processLoggingBroadcast, timeZone, discordAwarenessOutbox, discordAwarenessEmoji, discordAwarenessDeadlineMs, operatorLog = new OperatorLog(undefined)) {
|
|
874
|
+
this.operatorLog = operatorLog;
|
|
717
875
|
this.store = store;
|
|
718
876
|
this.ownsStore = ownsStore;
|
|
719
877
|
this.membrane = membrane;
|
|
@@ -774,6 +932,12 @@ export class AgentFramework {
|
|
|
774
932
|
catch {
|
|
775
933
|
// Already registered
|
|
776
934
|
}
|
|
935
|
+
try {
|
|
936
|
+
store.registerState({ id: DEFERRED_WRITES_ID, strategy: 'snapshot' });
|
|
937
|
+
}
|
|
938
|
+
catch {
|
|
939
|
+
// Already registered
|
|
940
|
+
}
|
|
777
941
|
try {
|
|
778
942
|
store.registerState({
|
|
779
943
|
id: INFERENCE_LOG_ID,
|
|
@@ -816,7 +980,17 @@ export class AgentFramework {
|
|
|
816
980
|
const discordAwarenessOutbox = discordAwarenessOutboxPath
|
|
817
981
|
? new DiscordAwarenessOutbox(discordAwarenessOutboxPath)
|
|
818
982
|
: null;
|
|
819
|
-
const
|
|
983
|
+
const operatorLogPath = config.operatorLogPath === false
|
|
984
|
+
? undefined
|
|
985
|
+
: config.operatorLogPath
|
|
986
|
+
?? (config.storePath ? defaultOperatorLogPath(config.storePath) : undefined);
|
|
987
|
+
if (operatorLogPath === undefined && config.operatorLogPath !== false) {
|
|
988
|
+
// An audit log that is silently off is worse than none: say so once.
|
|
989
|
+
console.error('[operator-log] disabled: no storePath and no operatorLogPath configured — ' +
|
|
990
|
+
'operator actions will only be visible as operator:action traces');
|
|
991
|
+
}
|
|
992
|
+
const framework = new AgentFramework(store, ownsStore, config.membrane, config.inferencePolicy ?? new DefaultInferencePolicy(), config.errorPolicy ?? new DefaultErrorPolicy(), config.syncIntervalMs ?? DEFAULT_SYNC_INTERVAL_MS, config.maintenanceIntervalMs ?? DEFAULT_MAINTENANCE_INTERVAL_MS, processLoggingPersist, processLoggingBroadcast, resolveTimeZone(config.timeZone), discordAwarenessOutbox, config.discordAwarenessEmoji ?? DEFAULT_DISCORD_AWARENESS_EMOJI, normalizeDiscordAwarenessDeadline(config.discordAwarenessDeadlineMs), new OperatorLog(operatorLogPath));
|
|
993
|
+
framework.providerHoldHook = config.providerHold;
|
|
820
994
|
// If an offline recovery process crashed after switching Chronicle but
|
|
821
995
|
// before committing its prepared marker batch, the active branch is the
|
|
822
996
|
// commit record. Promote it now; no quarantined content is read.
|
|
@@ -936,6 +1110,56 @@ export class AgentFramework {
|
|
|
936
1110
|
for (const [agentName, cap] of Object.entries(framework.readPersistedToolResultInlineCaps())) {
|
|
937
1111
|
framework.toolResultInlineMaxCharsOverride.set(agentName, cap);
|
|
938
1112
|
}
|
|
1113
|
+
// Restore persisted quiesce mode (issue #122) BEFORE initializeMcpl: the
|
|
1114
|
+
// flag must be set before any data-plane barrier completion can run, or
|
|
1115
|
+
// the startup funnel would open the data planes on a host that shut down
|
|
1116
|
+
// mid-maintenance. Staged connections boot with both planes closed, so a
|
|
1117
|
+
// quiesced boot needs no re-pause — completeMcplDataPlaneGate consults
|
|
1118
|
+
// the flag and holds data planes (control planes come up normally). The
|
|
1119
|
+
// gate already exists at this point, so the suppression is wired here too.
|
|
1120
|
+
//
|
|
1121
|
+
// Both records live OUTSIDE branch history (recovery/ files next to the
|
|
1122
|
+
// store): a historical rollback must not be able to erase the marker of
|
|
1123
|
+
// the very surgery it belongs to, nor orphan the writes deferred by it.
|
|
1124
|
+
framework.hostModePath = config.hostModePath
|
|
1125
|
+
?? (config.storePath ? join(config.storePath, 'recovery', 'host-mode.json') : undefined);
|
|
1126
|
+
framework.deferredWritesPath = config.deferredWritesPath
|
|
1127
|
+
?? (config.storePath ? join(config.storePath, 'recovery', 'deferred-writes.json') : undefined);
|
|
1128
|
+
{
|
|
1129
|
+
const hostMode = framework.readHostMode();
|
|
1130
|
+
// Deferred writes are recovered REGARDLESS of the mode flag: a crash
|
|
1131
|
+
// between a resume's flag clear and the end of its flush must not
|
|
1132
|
+
// strand accepted messages. Per-message acks mean only the remainder
|
|
1133
|
+
// replays. Serving boot with a remainder → flush it right now (no turn
|
|
1134
|
+
// is alive yet, so this is a safe boundary).
|
|
1135
|
+
const restoredWrites = framework.restorePersistedDeferredWrites();
|
|
1136
|
+
if (!hostMode?.quiesced && restoredWrites > 0) {
|
|
1137
|
+
console.error(`[host-mode] ${restoredWrites} deferred context write(s) found at a serving boot ` +
|
|
1138
|
+
`(an earlier resume did not finish its flush) — landing them now`);
|
|
1139
|
+
await framework.flushDeferredWrites('boot-recovery');
|
|
1140
|
+
}
|
|
1141
|
+
if (hostMode?.quiesced) {
|
|
1142
|
+
framework.quiesced = true;
|
|
1143
|
+
framework.quiesceReason = hostMode.reason;
|
|
1144
|
+
framework.quiescedAt = hostMode.since;
|
|
1145
|
+
framework.eventGate?.setQuiesced(true);
|
|
1146
|
+
console.error(`[host-mode] ============================================================\n` +
|
|
1147
|
+
`[host-mode] BOOTING QUIESCED (persisted${hostMode.reason ? `: ${hostMode.reason}` : ''}, ` +
|
|
1148
|
+
`since ${hostMode.since ? new Date(hostMode.since).toISOString() : 'unknown'}).\n` +
|
|
1149
|
+
`[host-mode] No wakes will start turns and MCPL data planes stay paused\n` +
|
|
1150
|
+
`[host-mode] until resume() — via host/command, the API server, or the\n` +
|
|
1151
|
+
`[host-mode] framework API. Wakes parked before the restart were in-memory\n` +
|
|
1152
|
+
`[host-mode] and are gone; recurring sources re-deliver on their own cadence.\n` +
|
|
1153
|
+
`[host-mode] ${restoredWrites} deferred context write(s) restored from the store;\n` +
|
|
1154
|
+
`[host-mode] they land at resume().\n` +
|
|
1155
|
+
`[host-mode] ============================================================`);
|
|
1156
|
+
framework.emitTrace({
|
|
1157
|
+
type: 'host:quiesced_boot',
|
|
1158
|
+
...(hostMode.reason ? { reason: hostMode.reason } : {}),
|
|
1159
|
+
...(hostMode.since !== undefined ? { since: hostMode.since } : {}),
|
|
1160
|
+
});
|
|
1161
|
+
}
|
|
1162
|
+
}
|
|
939
1163
|
// Initialize MCPL subsystems if configured
|
|
940
1164
|
if (config.mcplServers && config.mcplServers.length > 0) {
|
|
941
1165
|
// Validate tool prefixes: no collisions with module names or between servers
|
|
@@ -1040,6 +1264,9 @@ export class AgentFramework {
|
|
|
1040
1264
|
*/
|
|
1041
1265
|
async stop() {
|
|
1042
1266
|
this.running = false;
|
|
1267
|
+
// Flushed-but-unsynced deferred writes: sync and ack now, while the
|
|
1268
|
+
// store is still open, rather than leaving them to a reboot replay.
|
|
1269
|
+
this.ackDeferredWrites();
|
|
1043
1270
|
this.providerAdmissionClosed = true;
|
|
1044
1271
|
this.queue.close();
|
|
1045
1272
|
this.tuneOutCoordinator?.stop();
|
|
@@ -1177,6 +1404,15 @@ export class AgentFramework {
|
|
|
1177
1404
|
try {
|
|
1178
1405
|
return await run();
|
|
1179
1406
|
}
|
|
1407
|
+
catch (error) {
|
|
1408
|
+
// Compression can be the first call to find a spent quota (idle agent,
|
|
1409
|
+
// scheduled maintenance). Arm the host hold so later passes wait at the
|
|
1410
|
+
// gate instead of retrying into it; this pass still fails to its caller.
|
|
1411
|
+
const agent = this.agents.get(agentName);
|
|
1412
|
+
if (agent && error instanceof Error)
|
|
1413
|
+
this.holdProviderAcceleration(agent, error, undefined, true);
|
|
1414
|
+
throw error;
|
|
1415
|
+
}
|
|
1180
1416
|
finally {
|
|
1181
1417
|
gate.auxiliaryInFlight = Math.max(0, gate.auxiliaryInFlight - 1);
|
|
1182
1418
|
if (gate.auxiliaryInFlight === 0)
|
|
@@ -1206,11 +1442,51 @@ export class AgentFramework {
|
|
|
1206
1442
|
return a.agentName === b.agentName && a.reason === b.reason && a.source === b.source &&
|
|
1207
1443
|
a.timestamp === b.timestamp && a.channelId === b.channelId;
|
|
1208
1444
|
}
|
|
1209
|
-
|
|
1210
|
-
if (this.
|
|
1445
|
+
consultProviderHold(error, agentName) {
|
|
1446
|
+
if (!this.providerHoldHook)
|
|
1447
|
+
return undefined;
|
|
1448
|
+
try {
|
|
1449
|
+
const hold = this.providerHoldHook(error, agentName, { model: this.agents.get(agentName)?.model });
|
|
1450
|
+
if (!hold || !Number.isFinite(hold.holdMs) || hold.holdMs <= 0)
|
|
1451
|
+
return undefined;
|
|
1452
|
+
return { holdMs: Math.min(PROVIDER_ACCELERATION_MAX_COOLDOWN_MS, Math.max(1_000, hold.holdMs)),
|
|
1453
|
+
reason: typeof hold.reason === 'string' && hold.reason ? hold.reason : error.message };
|
|
1454
|
+
}
|
|
1455
|
+
catch (err) {
|
|
1456
|
+
console.error(`[provider-cooldown] providerHold hook threw for ${agentName}; treating as no hold:`, err);
|
|
1457
|
+
return undefined;
|
|
1458
|
+
}
|
|
1459
|
+
}
|
|
1460
|
+
/** A host hold whose slice expired: ask the host again before spending an
|
|
1461
|
+
* inference on it. True = extended in place (held requests stay held). */
|
|
1462
|
+
extendHostProviderHold(agentName, cooldown) {
|
|
1463
|
+
if (!cooldown.hostHoldError)
|
|
1464
|
+
return false;
|
|
1465
|
+
const hold = this.consultProviderHold(cooldown.hostHoldError, agentName);
|
|
1466
|
+
if (!hold)
|
|
1467
|
+
return false;
|
|
1468
|
+
clearTimeout(cooldown.timer);
|
|
1469
|
+
cooldown.until = Date.now() + hold.holdMs;
|
|
1470
|
+
cooldown.timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agentName), hold.holdMs);
|
|
1471
|
+
cooldown.timer.unref?.();
|
|
1472
|
+
if (hold.reason !== cooldown.reason) {
|
|
1473
|
+
cooldown.reason = hold.reason;
|
|
1474
|
+
console.error(`[provider-cooldown] agent=${agentName} host hold extended — ${hold.reason}`);
|
|
1475
|
+
}
|
|
1476
|
+
return true;
|
|
1477
|
+
}
|
|
1478
|
+
holdProviderAcceleration(agent, error, trigger, auxiliary = false) {
|
|
1479
|
+
if (this.ephemeralRuns.has(agent.name) || this.conversationAgentHomes.has(agent.name))
|
|
1480
|
+
return false;
|
|
1481
|
+
// The host knows more than the message wording does: ask it first. An
|
|
1482
|
+
// auxiliary failure arms host holds only — the built-in acceleration
|
|
1483
|
+
// cooldown stays a primary-path mechanism.
|
|
1484
|
+
const hostHold = this.consultProviderHold(error, agent.name);
|
|
1485
|
+
const acceleration = !hostHold && !auxiliary && isOrganizationAccelerationRateLimit(error);
|
|
1486
|
+
if (!acceleration && !hostHold)
|
|
1211
1487
|
return false;
|
|
1212
1488
|
const now = Date.now();
|
|
1213
|
-
const delayMs = this.accelerationCooldownMs(agent.name, error);
|
|
1489
|
+
const delayMs = hostHold ? hostHold.holdMs : this.accelerationCooldownMs(agent.name, error);
|
|
1214
1490
|
const existing = this.providerAccelerationCooldowns.get(agent.name);
|
|
1215
1491
|
const held = existing?.heldRequests ?? [];
|
|
1216
1492
|
if (trigger && !held.some((r) => this.sameInferenceRequest(r, trigger)))
|
|
@@ -1220,13 +1496,20 @@ export class AgentFramework {
|
|
|
1220
1496
|
const timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agent.name), delayMs);
|
|
1221
1497
|
timer.unref?.();
|
|
1222
1498
|
this.providerAccelerationCooldowns.set(agent.name, { startedAt: existing?.startedAt ?? now,
|
|
1223
|
-
until: now + delayMs, timer, heldRequests: held, reason: error.message,
|
|
1224
|
-
failures: (existing?.failures ?? 0) + 1 })
|
|
1499
|
+
until: now + delayMs, timer, heldRequests: held, reason: hostHold?.reason ?? error.message,
|
|
1500
|
+
failures: (existing?.failures ?? 0) + 1, ...(hostHold ? { hostHoldError: error } : {}),
|
|
1501
|
+
...(auxiliary && (!existing || existing.auxiliaryOrigin) ? { auxiliaryOrigin: true } : {}) });
|
|
1502
|
+
if (auxiliary && !existing) {
|
|
1503
|
+
console.error(`[provider-cooldown] agent=${agent.name} host hold (${hostHold.reason}) from an auxiliary call — ` +
|
|
1504
|
+
`holding primary/auxiliary for ${delayMs}ms`);
|
|
1505
|
+
return true;
|
|
1506
|
+
}
|
|
1225
1507
|
const gate = this.providerGate(agent.name);
|
|
1226
1508
|
gate.primaryPending = true;
|
|
1227
1509
|
this.providerAccelerationRecoveries.set(agent.name, { startedAt: existing?.startedAt ?? now,
|
|
1228
1510
|
failures: (existing?.failures ?? 0) + 1, heldRequests: held.length, reason: error.message });
|
|
1229
|
-
console.error(`[provider-cooldown] agent=${agent.name}
|
|
1511
|
+
console.error(`[provider-cooldown] agent=${agent.name} ` +
|
|
1512
|
+
`${hostHold ? `host hold (${hostHold.reason})` : 'organization acceleration 429'} — ` +
|
|
1230
1513
|
`holding primary/auxiliary for ${delayMs}ms; ${held.length} request(s) retained`);
|
|
1231
1514
|
return true;
|
|
1232
1515
|
}
|
|
@@ -1234,8 +1517,16 @@ export class AgentFramework {
|
|
|
1234
1517
|
const cooldown = this.providerAccelerationCooldowns.get(agentName);
|
|
1235
1518
|
if (!cooldown)
|
|
1236
1519
|
return;
|
|
1520
|
+
if (this.extendHostProviderHold(agentName, cooldown))
|
|
1521
|
+
return;
|
|
1237
1522
|
clearTimeout(cooldown.timer);
|
|
1238
1523
|
this.providerAccelerationCooldowns.delete(agentName);
|
|
1524
|
+
if (cooldown.auxiliaryOrigin && cooldown.heldRequests.length === 0) {
|
|
1525
|
+
// Nobody asked for an inference while parked: just reopen the gate.
|
|
1526
|
+
console.error(`[provider-cooldown] agent=${agentName} auxiliary hold released after ${Date.now() - cooldown.startedAt}ms`);
|
|
1527
|
+
this.flushAuxiliaryAdmission(agentName);
|
|
1528
|
+
return;
|
|
1529
|
+
}
|
|
1239
1530
|
const recovery = this.providerAccelerationRecoveries.get(agentName);
|
|
1240
1531
|
if (recovery) {
|
|
1241
1532
|
recovery.releasedAt = Date.now();
|
|
@@ -1766,10 +2057,60 @@ export class AgentFramework {
|
|
|
1766
2057
|
const agent = this.agents.get(agentName);
|
|
1767
2058
|
if (!agent)
|
|
1768
2059
|
throw new Error(`Unknown agent: ${agentName}`);
|
|
2060
|
+
// Feasibility preflight — only a budget-bearing patch can wedge, and only
|
|
2061
|
+
// on the immediate path (a paced descent leaves the compile budget alone
|
|
2062
|
+
// and converges only as far as the folded floor allows). Boot restore
|
|
2063
|
+
// deliberately bypasses this wrapper (createAgent → restoreRuntimeSettings),
|
|
2064
|
+
// so a persisted-but-now-infeasible budget can never brick startup.
|
|
2065
|
+
if (patch.contextBudgetTokens !== undefined) {
|
|
2066
|
+
const preview = this.previewAgentRuntimeSettings(agentName, patch);
|
|
2067
|
+
if (preview.available && preview.effective) {
|
|
2068
|
+
const e = preview.effective;
|
|
2069
|
+
// Block only a LOWERING that doesn't fit. A no-op rewrite or an
|
|
2070
|
+
// increase on an already-over-floor agent never makes things worse —
|
|
2071
|
+
// refusing those would throw on exactly the wedged population the
|
|
2072
|
+
// guard exists to protect (and block stepwise remediation).
|
|
2073
|
+
const lowering = patch.contextBudgetTokens < preview.liveBudgetTokens;
|
|
2074
|
+
if (preview.path === 'immediate' && !e.fits && lowering) {
|
|
2075
|
+
const msg = `[budget-preflight] contextBudgetTokens=${patch.contextBudgetTokens} would NOT fit ` +
|
|
2076
|
+
`${agentName}: the folded floor renders ${e.finalTokens} tokens against hard budget ` +
|
|
2077
|
+
`${e.budgetTokens} (head=${e.headTokens ?? '?'} tail=${e.tailTokens ?? '?'} ` +
|
|
2078
|
+
`middle=${e.middleTokens ?? '?'}, deepest L${e.deepestLevel ?? '?'}` +
|
|
2079
|
+
`${e.exhausted ? ', picker exhausted' : ''}). Every compile at this budget would ` +
|
|
2080
|
+
`OverBudget-wedge the agent. Lower the floor first (drain compression quarantine / ` +
|
|
2081
|
+
`advance the merge ladder), use a paced descent (no \`immediate\`), or pass allowInfeasible.`;
|
|
2082
|
+
if (!opts?.allowInfeasible)
|
|
2083
|
+
throw new BudgetPreflightError(msg, preview);
|
|
2084
|
+
console.warn(`${msg} — applying anyway (allowInfeasible).`);
|
|
2085
|
+
}
|
|
2086
|
+
else if (preview.path === 'immediate' && !e.fits) {
|
|
2087
|
+
console.warn(`[budget-preflight] ${agentName} remains over the folded floor at ` +
|
|
2088
|
+
`${patch.contextBudgetTokens} (floor renders ${e.finalTokens}); this patch does ` +
|
|
2089
|
+
`not lower the budget, so applying — drain quarantine / advance merges to clear it.`);
|
|
2090
|
+
}
|
|
2091
|
+
else if (preview.path === 'paced' && preview.advisory && !preview.advisory.fits) {
|
|
2092
|
+
console.warn(`[budget-preflight] paced descent for ${agentName} targets ` +
|
|
2093
|
+
`${preview.advisory.targetTokens} tokens, below the current folded floor — the ` +
|
|
2094
|
+
`transition will converge only as far as the floor allows (drain quarantine / ` +
|
|
2095
|
+
`advance merges to go lower). Applying; the live compile budget is unchanged.`);
|
|
2096
|
+
}
|
|
2097
|
+
}
|
|
2098
|
+
else if (preview.reason) {
|
|
2099
|
+
console.warn(`[budget-preflight] preview unavailable for ${agentName} (${preview.reason}); ` +
|
|
2100
|
+
`applying without preflight`);
|
|
2101
|
+
}
|
|
2102
|
+
}
|
|
1769
2103
|
const result = agent.updateRuntimeSettings(patch);
|
|
1770
2104
|
if (opts?.persist !== false) {
|
|
1771
2105
|
this.persistAgentRuntimeSettings(agentName, agent.getRuntimeSettingsOverrides());
|
|
1772
2106
|
}
|
|
2107
|
+
this.recordOperatorAction({
|
|
2108
|
+
kind: 'settings-update',
|
|
2109
|
+
agent: agentName,
|
|
2110
|
+
...(opts?.requester ? { requester: opts.requester } : {}),
|
|
2111
|
+
...(opts?.note ? { note: opts.note } : {}),
|
|
2112
|
+
params: { patch: { ...patch }, persist: opts?.persist !== false },
|
|
2113
|
+
});
|
|
1773
2114
|
return result;
|
|
1774
2115
|
}
|
|
1775
2116
|
resetAgentRuntimeSettings(agentName, keys, opts) {
|
|
@@ -1780,16 +2121,616 @@ export class AgentFramework {
|
|
|
1780
2121
|
if (opts?.persist !== false) {
|
|
1781
2122
|
this.persistAgentRuntimeSettings(agentName, agent.getRuntimeSettingsOverrides());
|
|
1782
2123
|
}
|
|
2124
|
+
this.recordOperatorAction({
|
|
2125
|
+
kind: 'settings-reset',
|
|
2126
|
+
agent: agentName,
|
|
2127
|
+
...(opts?.requester ? { requester: opts.requester } : {}),
|
|
2128
|
+
...(opts?.note ? { note: opts.note } : {}),
|
|
2129
|
+
params: { keys: keys ?? 'all', persist: opts?.persist !== false },
|
|
2130
|
+
});
|
|
1783
2131
|
return result;
|
|
1784
2132
|
}
|
|
1785
|
-
cancelAgentRuntimeSettingsTransition(agentName) {
|
|
2133
|
+
cancelAgentRuntimeSettingsTransition(agentName, opts) {
|
|
1786
2134
|
const agent = this.agents.get(agentName);
|
|
1787
2135
|
if (!agent)
|
|
1788
2136
|
throw new Error(`Unknown agent: ${agentName}`);
|
|
1789
2137
|
const result = agent.cancelRuntimeSettingsTransition();
|
|
1790
2138
|
this.persistAgentRuntimeSettings(agentName, agent.getRuntimeSettingsOverrides());
|
|
2139
|
+
this.recordOperatorAction({
|
|
2140
|
+
kind: 'settings-cancel-transition',
|
|
2141
|
+
agent: agentName,
|
|
2142
|
+
...(opts?.requester ? { requester: opts.requester } : {}),
|
|
2143
|
+
});
|
|
1791
2144
|
return result;
|
|
1792
2145
|
}
|
|
2146
|
+
// -------------------------------------------------------------------------
|
|
2147
|
+
// Host quiesce / maintenance mode (issue #122)
|
|
2148
|
+
//
|
|
2149
|
+
// Pause the inference thread and MCPL data planes while keeping the
|
|
2150
|
+
// framework + context managers + membrane loaded and hot, so maintenance
|
|
2151
|
+
// (compression ticks, refold, quarantine drains, budget descents) runs
|
|
2152
|
+
// through the exact machinery the agent uses live — with live config, live
|
|
2153
|
+
// tool definitions, and full llm-calls logging — instead of offline rigs
|
|
2154
|
+
// that re-derive all of it and drift.
|
|
2155
|
+
//
|
|
2156
|
+
// While quiesced: runLoop, store sync, the maintenance timer, and the
|
|
2157
|
+
// liveness watchdog all keep running. Wakes park in pendingRequests
|
|
2158
|
+
// (coalesced per reason) and fire at resume; MCPL events buffer on the
|
|
2159
|
+
// paused data planes; gate debounces buffer in the gate. Background
|
|
2160
|
+
// code-execution scripts are NOT stopped (they hold no turn token) — they
|
|
2161
|
+
// are reported in the status so the operator sees what still acts.
|
|
2162
|
+
//
|
|
2163
|
+
// NOTE: do not call quiesce() from inside a queue event handler — the drain
|
|
2164
|
+
// wait depends on the event loop continuing to run.
|
|
2165
|
+
// -------------------------------------------------------------------------
|
|
2166
|
+
/**
|
|
2167
|
+
* Wakes that passed the scheduler and now own provider admission while
|
|
2168
|
+
* waiting for an in-flight auxiliary call to settle — no turn token yet,
|
|
2169
|
+
* so `activeTurnTokens` alone would call the host drained while a turn is
|
|
2170
|
+
* one promise-resolution away from starting. The admission continuation
|
|
2171
|
+
* rechecks quiesce and requeues, but until it runs the wake is live.
|
|
2172
|
+
*/
|
|
2173
|
+
parkedAdmissionCount() {
|
|
2174
|
+
let n = 0;
|
|
2175
|
+
for (const [agentName, gate] of this.providerGates) {
|
|
2176
|
+
if (gate.primaryDepth > 0 && !this.activeTurnTokens.has(agentName))
|
|
2177
|
+
n++;
|
|
2178
|
+
}
|
|
2179
|
+
return n;
|
|
2180
|
+
}
|
|
2181
|
+
getHostModeStatus() {
|
|
2182
|
+
const parkedAdmissions = this.parkedAdmissionCount();
|
|
2183
|
+
return {
|
|
2184
|
+
quiesced: this.quiesced,
|
|
2185
|
+
...(this.quiesceReason ? { reason: this.quiesceReason } : {}),
|
|
2186
|
+
...(this.quiescedAt !== undefined ? { since: this.quiescedAt } : {}),
|
|
2187
|
+
drained: this.activeTurnTokens.size === 0 && parkedAdmissions === 0,
|
|
2188
|
+
activeTurns: this.activeTurnTokens.size,
|
|
2189
|
+
parkedAdmissions,
|
|
2190
|
+
gatedRequests: this.pendingRequests.filter((r) => !isTurnContinuation(r.reason)).length,
|
|
2191
|
+
backgroundScripts: [...this.backgroundScripts.values()]
|
|
2192
|
+
.filter((record) => record.status === 'running').length,
|
|
2193
|
+
deferredWrites: this.deferredMessages.length,
|
|
2194
|
+
...(this.lastUnabandonable.length > 0 ? { unabandonable: [...this.lastUnabandonable] } : {}),
|
|
2195
|
+
};
|
|
2196
|
+
}
|
|
2197
|
+
/** Drain window bounds shared by every ingress (framework API, WS, HTTP,
|
|
2198
|
+
* host/command). Below the floor a drain is meaningless; above the ceiling
|
|
2199
|
+
* an awaited call would poll for days. */
|
|
2200
|
+
static QUIESCE_TIMEOUT_MIN_MS = 1_000;
|
|
2201
|
+
static QUIESCE_TIMEOUT_MAX_MS = 600_000;
|
|
2202
|
+
static QUIESCE_TIMEOUT_DEFAULT_MS = 120_000;
|
|
2203
|
+
/**
|
|
2204
|
+
* Enter quiesce mode: persist the flag, suppress gate deliveries, pause all
|
|
2205
|
+
* MCPL data planes, and wait for in-flight turns to settle (turn-alive is
|
|
2206
|
+
* `activeTurnTokens`, which spans dequeue → settled teardown). Idempotent.
|
|
2207
|
+
*
|
|
2208
|
+
* On drain timeout the host STAYS quiesced (`drained: false` in the
|
|
2209
|
+
* result); with `abandon: true` the undrained streams are cancelled via a
|
|
2210
|
+
* dedicated cancel kind that settles the turn without feeding the
|
|
2211
|
+
* inference-failure accounting.
|
|
2212
|
+
*/
|
|
2213
|
+
async quiesce(opts) {
|
|
2214
|
+
if (this.quiesced) {
|
|
2215
|
+
// Idempotent — EXCEPT abandon escalation: a first quiesce that timed
|
|
2216
|
+
// out undrained must be escalatable with a second quiesce({abandon})
|
|
2217
|
+
// without resume()+re-quiesce (which would reopen data planes and
|
|
2218
|
+
// release parked wakes mid-surgery).
|
|
2219
|
+
if (opts?.abandon && this.activeTurnTokens.size > 0) {
|
|
2220
|
+
const unabandonable = await this.abandonActiveTurns();
|
|
2221
|
+
this.emitTrace({
|
|
2222
|
+
type: 'host:quiesce',
|
|
2223
|
+
...(this.quiesceReason ? { reason: this.quiesceReason } : {}),
|
|
2224
|
+
drained: this.activeTurnTokens.size === 0,
|
|
2225
|
+
activeTurns: this.activeTurnTokens.size,
|
|
2226
|
+
abandoned: true,
|
|
2227
|
+
...(unabandonable.length > 0 ? { unabandonable } : {}),
|
|
2228
|
+
});
|
|
2229
|
+
}
|
|
2230
|
+
return this.getHostModeStatus();
|
|
2231
|
+
}
|
|
2232
|
+
this.quiesced = true;
|
|
2233
|
+
this.quiesceReason = opts?.reason;
|
|
2234
|
+
this.quiescedAt = Date.now();
|
|
2235
|
+
this.lastUnabandonable = [];
|
|
2236
|
+
this.persistHostMode({
|
|
2237
|
+
quiesced: true,
|
|
2238
|
+
...(opts?.reason ? { reason: opts.reason } : {}),
|
|
2239
|
+
since: this.quiescedAt,
|
|
2240
|
+
});
|
|
2241
|
+
console.error(`[host-mode] quiescing${opts?.reason ? ` (${opts.reason})` : ''}: parking wakes, ` +
|
|
2242
|
+
`pausing MCPL data planes, draining ${this.activeTurnTokens.size} in-flight turn(s)`);
|
|
2243
|
+
this.eventGate?.setQuiesced(true);
|
|
2244
|
+
for (const connection of this.mcplServerRegistry?.getAllServers() ?? []) {
|
|
2245
|
+
connection.pauseDataPlane();
|
|
2246
|
+
}
|
|
2247
|
+
// Clamp HERE, for every ingress. NaN-proof too: timeoutMs arrives via
|
|
2248
|
+
// Number()/blind casts, and Math.max(1_000, NaN) is NaN — which would
|
|
2249
|
+
// silently collapse the drain window to zero (`Date.now() < NaN` is
|
|
2250
|
+
// false). The ceiling matters because this call is awaited: an
|
|
2251
|
+
// unbounded value polls the event loop for as long as it says.
|
|
2252
|
+
const rawTimeout = opts?.timeoutMs;
|
|
2253
|
+
const timeoutMs = typeof rawTimeout === 'number' && Number.isFinite(rawTimeout)
|
|
2254
|
+
? Math.max(AgentFramework.QUIESCE_TIMEOUT_MIN_MS, Math.min(AgentFramework.QUIESCE_TIMEOUT_MAX_MS, Math.floor(rawTimeout)))
|
|
2255
|
+
: AgentFramework.QUIESCE_TIMEOUT_DEFAULT_MS;
|
|
2256
|
+
const deadline = Date.now() + timeoutMs;
|
|
2257
|
+
// Drain = no turn token AND no wake parked on provider admission (the
|
|
2258
|
+
// latter becomes a turn the moment its auxiliary settles unless the
|
|
2259
|
+
// continuation's quiesce recheck requeues it — wait for that to happen).
|
|
2260
|
+
while (this.quiesced &&
|
|
2261
|
+
(this.activeTurnTokens.size > 0 || this.parkedAdmissionCount() > 0) &&
|
|
2262
|
+
Date.now() < deadline) {
|
|
2263
|
+
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
2264
|
+
}
|
|
2265
|
+
// A concurrent resume() ended this window while we waited. Nothing here
|
|
2266
|
+
// may act any more — an abandon now would cancel a turn that started
|
|
2267
|
+
// legitimately AFTER the resume, and the trace/log would describe a mode
|
|
2268
|
+
// the host is no longer in.
|
|
2269
|
+
if (!this.quiesced) {
|
|
2270
|
+
console.error('[host-mode] quiesce superseded by a concurrent resume during drain');
|
|
2271
|
+
return this.getHostModeStatus();
|
|
2272
|
+
}
|
|
2273
|
+
let abandoned = false;
|
|
2274
|
+
let unabandonable = [];
|
|
2275
|
+
if (this.activeTurnTokens.size > 0 && opts?.abandon) {
|
|
2276
|
+
abandoned = true;
|
|
2277
|
+
unabandonable = await this.abandonActiveTurns();
|
|
2278
|
+
if (!this.quiesced)
|
|
2279
|
+
return this.getHostModeStatus();
|
|
2280
|
+
}
|
|
2281
|
+
const status = this.getHostModeStatus();
|
|
2282
|
+
this.emitTrace({
|
|
2283
|
+
type: 'host:quiesce',
|
|
2284
|
+
...(this.quiesceReason ? { reason: this.quiesceReason } : {}),
|
|
2285
|
+
drained: status.drained,
|
|
2286
|
+
activeTurns: status.activeTurns,
|
|
2287
|
+
...(status.parkedAdmissions > 0 ? { parkedAdmissions: status.parkedAdmissions } : {}),
|
|
2288
|
+
...(abandoned ? { abandoned: true } : {}),
|
|
2289
|
+
...(unabandonable.length > 0 ? { unabandonable } : {}),
|
|
2290
|
+
});
|
|
2291
|
+
console.error(status.drained
|
|
2292
|
+
? `[host-mode] quiesced — drained, ${status.gatedRequests} wake(s) parked`
|
|
2293
|
+
: `[host-mode] quiesced but drain ${abandoned ? 'needed abandon and' : 'timed out —'} ` +
|
|
2294
|
+
`${status.activeTurns} turn(s) still alive` +
|
|
2295
|
+
(status.parkedAdmissions > 0
|
|
2296
|
+
? `, ${status.parkedAdmissions} wake(s) parked on provider admission`
|
|
2297
|
+
: ''));
|
|
2298
|
+
return status;
|
|
2299
|
+
}
|
|
2300
|
+
/** Cancel every turn-alive stream via the quiesce_abandoned kind and give
|
|
2301
|
+
* teardown a bounded grace to settle. Shared by the drain-timeout path and
|
|
2302
|
+
* the already-quiesced escalation path. */
|
|
2303
|
+
async abandonActiveTurns() {
|
|
2304
|
+
const unabandonable = [];
|
|
2305
|
+
for (const agentName of [...this.activeTurnTokens.keys()]) {
|
|
2306
|
+
const agent = this.agents.get(agentName);
|
|
2307
|
+
const state = agent?.state;
|
|
2308
|
+
const stream = state && 'stream' in state ? state.stream : undefined;
|
|
2309
|
+
if (agent && stream) {
|
|
2310
|
+
console.error(`[host-mode] abandoning in-flight turn for ${agentName}`);
|
|
2311
|
+
this.frameworkCancelledStreams.set(`${agent.name}:${agent.streamId}`, 'quiesce_abandoned');
|
|
2312
|
+
stream.cancel();
|
|
2313
|
+
}
|
|
2314
|
+
else {
|
|
2315
|
+
// Token held, no stream to cancel: the turn is between dequeue and
|
|
2316
|
+
// stream registration (hooks/compile), or it is a puppet tool turn,
|
|
2317
|
+
// which never has one. Say so — silence here reads as "abandon acted
|
|
2318
|
+
// and teardown is slow", which is a different operator decision.
|
|
2319
|
+
unabandonable.push(agentName);
|
|
2320
|
+
console.error(`[host-mode] cannot abandon turn for ${agentName}: turn token held with no stream ` +
|
|
2321
|
+
`(status=${state?.status ?? 'unknown'}) — it must settle on its own`);
|
|
2322
|
+
}
|
|
2323
|
+
}
|
|
2324
|
+
// Bounded grace for the cancelled streams' teardown to settle.
|
|
2325
|
+
const grace = Date.now() + 10_000;
|
|
2326
|
+
while (this.quiesced && this.activeTurnTokens.size > 0 && Date.now() < grace) {
|
|
2327
|
+
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
2328
|
+
}
|
|
2329
|
+
// Only turns still alive after the grace are worth reporting.
|
|
2330
|
+
this.lastUnabandonable = unabandonable.filter((name) => this.activeTurnTokens.has(name));
|
|
2331
|
+
return this.lastUnabandonable;
|
|
2332
|
+
}
|
|
2333
|
+
/**
|
|
2334
|
+
* Leave quiesce mode. Gates on a FRESH feasibility preview of every agent's
|
|
2335
|
+
* CURRENT settings — maintenance may have moved the folded floor and the
|
|
2336
|
+
* branch generation, so verdicts are computed at resume time, never reused.
|
|
2337
|
+
* Throws ResumeBlockedError (with all failing verdicts) unless `force`.
|
|
2338
|
+
* Unavailable previews warn and pass — refusing to resume because the
|
|
2339
|
+
* strategy cannot preview would hold hosts hostage to a diagnostic.
|
|
2340
|
+
*/
|
|
2341
|
+
async resume(opts) {
|
|
2342
|
+
if (!this.quiesced)
|
|
2343
|
+
return this.getHostModeStatus();
|
|
2344
|
+
// The canonical operator flow is `maintain; resume`, and a timer-driven
|
|
2345
|
+
// pass may still be mid-flight: previewContext refuses to overlap one
|
|
2346
|
+
// ("already running"), which used to degrade the gate to "no opinion".
|
|
2347
|
+
// Wait for the pass first, then retry a few times for the case where a
|
|
2348
|
+
// fresh pass started in between.
|
|
2349
|
+
if (this.maintenancePass) {
|
|
2350
|
+
await this.maintenancePass.catch(() => { });
|
|
2351
|
+
}
|
|
2352
|
+
const failing = [];
|
|
2353
|
+
for (const agentName of this.agents.keys()) {
|
|
2354
|
+
let preview = this.previewAgentRuntimeSettings(agentName);
|
|
2355
|
+
for (let attempt = 0; attempt < 8 && !preview.available && preview.reason === 'preview_in_flight'; attempt++) {
|
|
2356
|
+
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
2357
|
+
if (this.maintenancePass)
|
|
2358
|
+
await this.maintenancePass.catch(() => { });
|
|
2359
|
+
preview = this.previewAgentRuntimeSettings(agentName);
|
|
2360
|
+
}
|
|
2361
|
+
if (preview.available && preview.effective && !preview.effective.fits) {
|
|
2362
|
+
failing.push({ agentName, preview });
|
|
2363
|
+
}
|
|
2364
|
+
else if (!preview.available && preview.reason !== 'no_preview_support') {
|
|
2365
|
+
console.warn(`[host-mode] resume: feasibility preview unavailable for ${agentName} ` +
|
|
2366
|
+
`(${preview.reason}) — proceeding without a verdict`);
|
|
2367
|
+
}
|
|
2368
|
+
}
|
|
2369
|
+
if (failing.length > 0 && !opts?.force) {
|
|
2370
|
+
const detail = failing.map(({ agentName, preview }) => `${agentName}: folded floor ${preview.effective.finalTokens} > hard budget ` +
|
|
2371
|
+
`${preview.effective.budgetTokens}` +
|
|
2372
|
+
(preview.transition === 'blocked'
|
|
2373
|
+
? ` (transition blocked: ${preview.transitionReason ?? 'unknown'})`
|
|
2374
|
+
: '')).join('; ');
|
|
2375
|
+
throw new ResumeBlockedError(`resume refused: returning to service would OverBudget-wedge — ${detail}. ` +
|
|
2376
|
+
`Drain compression quarantine / advance the merge ladder to lower the floor ` +
|
|
2377
|
+
`(maintenanceTick()), or resume({ force: true }).`, failing);
|
|
2378
|
+
}
|
|
2379
|
+
if (failing.length > 0) {
|
|
2380
|
+
console.warn(`[host-mode] resume FORCED past ${failing.length} failing feasibility verdict(s)`);
|
|
2381
|
+
}
|
|
2382
|
+
const releasedRequests = this.pendingRequests.length;
|
|
2383
|
+
// In-memory first: addMessage must stop deferring so the flush below can
|
|
2384
|
+
// land. The DURABLE flag is cleared only after the flush completes — a
|
|
2385
|
+
// crash in between then boots quiesced with the un-acked remainder
|
|
2386
|
+
// restored (and a serving boot recovers any remainder anyway).
|
|
2387
|
+
this.quiesced = false;
|
|
2388
|
+
this.quiesceReason = undefined;
|
|
2389
|
+
this.quiescedAt = undefined;
|
|
2390
|
+
this.lastUnabandonable = [];
|
|
2391
|
+
this.eventGate?.setQuiesced(false);
|
|
2392
|
+
try {
|
|
2393
|
+
// Flush context writes deferred by the quiesce window (module events,
|
|
2394
|
+
// api message.send) — with no turn alive there is no other flush
|
|
2395
|
+
// point. Per-message try/catch and per-message durable ack (see
|
|
2396
|
+
// flushDeferredWrites); nothing here may reach the plane reopen below.
|
|
2397
|
+
await this.flushDeferredWrites('resume');
|
|
2398
|
+
this.persistHostMode(null);
|
|
2399
|
+
}
|
|
2400
|
+
finally {
|
|
2401
|
+
// Reopen MCPL data planes through the existing barrier funnel — NOT a
|
|
2402
|
+
// bespoke ready() loop. The funnel inherits completeMcplDataPlaneGate's
|
|
2403
|
+
// nested-install guard (a flushed tools-list-changed can install a
|
|
2404
|
+
// newer barrier mid-flush), drains any awareness work accumulated
|
|
2405
|
+
// during the window before opening, and replaces a stale failed
|
|
2406
|
+
// barrier by identity. In `finally` so that nothing above can leave
|
|
2407
|
+
// the host un-quiesced with every data plane still paused.
|
|
2408
|
+
if (this.mcplServerRegistry) {
|
|
2409
|
+
const barrier = this.installMcplDataPlaneGate();
|
|
2410
|
+
this.releaseMcplDataPlaneGate(barrier);
|
|
2411
|
+
try {
|
|
2412
|
+
await barrier.promise;
|
|
2413
|
+
this.completeMcplDataPlaneGate(barrier);
|
|
2414
|
+
}
|
|
2415
|
+
catch (error) {
|
|
2416
|
+
// The host IS resumed — don't rethrow. failMcplDataPlaneGate
|
|
2417
|
+
// recycles the connections, and their reconnect flows re-run the
|
|
2418
|
+
// funnel with quiesced=false, self-healing the data planes.
|
|
2419
|
+
await this.failMcplDataPlaneGate(barrier, 'quiesce resume', error);
|
|
2420
|
+
}
|
|
2421
|
+
}
|
|
2422
|
+
this.emitTrace({
|
|
2423
|
+
type: 'host:resume',
|
|
2424
|
+
...(opts?.force && failing.length > 0 ? { forced: true } : {}),
|
|
2425
|
+
releasedRequests,
|
|
2426
|
+
});
|
|
2427
|
+
// NOTE: parked wakes and gate buffers are in-memory only — a restart
|
|
2428
|
+
// mid-quiesce boots quiesced but with these queues empty (sources like
|
|
2429
|
+
// heartbeats re-deliver on their own cadence; one-shot wakes are lost).
|
|
2430
|
+
// Deferred context WRITES are persisted while quiesced and survive.
|
|
2431
|
+
console.error(`[host-mode] resumed — ${releasedRequests} parked wake(s) released`);
|
|
2432
|
+
}
|
|
2433
|
+
return this.getHostModeStatus();
|
|
2434
|
+
}
|
|
2435
|
+
/**
|
|
2436
|
+
* Land every deferred write whose target has no turn alive, one at a time,
|
|
2437
|
+
* acknowledging each durably as it lands (the persisted queue is rewritten
|
|
2438
|
+
* after every message). A crash mid-flush therefore replays exactly the
|
|
2439
|
+
* un-acked remainder at the next boot — no loss, no duplicates. A write
|
|
2440
|
+
* the store REJECTS is logged and acked too: replaying a poison message
|
|
2441
|
+
* forever would wedge every later boot on it.
|
|
2442
|
+
*
|
|
2443
|
+
* Per-target: an agent whose turn is still alive keeps ITS messages
|
|
2444
|
+
* deferred for its own turn boundary; everyone else's flush now.
|
|
2445
|
+
*/
|
|
2446
|
+
async flushDeferredWrites(label) {
|
|
2447
|
+
if (this.deferredMessages.length === 0) {
|
|
2448
|
+
this.persistDeferredWrites();
|
|
2449
|
+
return;
|
|
2450
|
+
}
|
|
2451
|
+
const keep = [];
|
|
2452
|
+
const flush = [];
|
|
2453
|
+
for (const msg of this.deferredMessages) {
|
|
2454
|
+
const target = msg.forAgent ?? this.primaryAgentName;
|
|
2455
|
+
(target && this.activeTurnTokens.has(target) ? keep : flush).push(msg);
|
|
2456
|
+
}
|
|
2457
|
+
// Hand-off: the batch leaves the pending queue and enters the un-acked
|
|
2458
|
+
// set. The DURABLE queue (pending + un-acked) is unchanged by this, so a
|
|
2459
|
+
// crash anywhere below replays the whole batch, and the deferredWriteId
|
|
2460
|
+
// stamped on each stored message lets boot skip the ones that did land.
|
|
2461
|
+
flush.sort(bySeq);
|
|
2462
|
+
this.deferredMessages = keep;
|
|
2463
|
+
this.handOffDeferredWrites(flush);
|
|
2464
|
+
let stored = 0;
|
|
2465
|
+
for (const msg of flush) {
|
|
2466
|
+
try {
|
|
2467
|
+
this.addMessage(msg.participant, msg.content, msg.metadata, {
|
|
2468
|
+
deferredWriteId: msg.id,
|
|
2469
|
+
...(msg.forAgent ? { forAgent: msg.forAgent } : {}),
|
|
2470
|
+
});
|
|
2471
|
+
stored++;
|
|
2472
|
+
}
|
|
2473
|
+
catch (err) {
|
|
2474
|
+
console.error(`[host-mode] ${label}: failed to store a deferred context write ` +
|
|
2475
|
+
`(participant=${msg.participant}, forAgent=${msg.forAgent ?? 'primary'}):`, err);
|
|
2476
|
+
}
|
|
2477
|
+
}
|
|
2478
|
+
// Ack = sync the chronicle FIRST, then rewrite the durable queue without
|
|
2479
|
+
// the batch. Never the other way round.
|
|
2480
|
+
this.ackDeferredWrites();
|
|
2481
|
+
console.error(`[host-mode] ${label}: flushed ${stored}/${flush.length} deferred context write(s)` +
|
|
2482
|
+
(keep.length > 0 ? `, ${keep.length} kept for a still-alive turn` : ''));
|
|
2483
|
+
}
|
|
2484
|
+
/**
|
|
2485
|
+
* Acknowledge every deferred write that has been handed to a context
|
|
2486
|
+
* manager: sync the chronicle so the appended slots are durable, and only
|
|
2487
|
+
* then drop them from the recovery queue. If the sync fails they stay in
|
|
2488
|
+
* the durable queue (the next ack, or stop(), retries); a reboot in that
|
|
2489
|
+
* state replays them, deduplicated by `deferredWriteId` for any that did
|
|
2490
|
+
* reach disk. Idempotent and cheap when nothing is un-acked.
|
|
2491
|
+
*/
|
|
2492
|
+
ackDeferredWrites() {
|
|
2493
|
+
if (this.unackedDeferredWrites.length === 0)
|
|
2494
|
+
return;
|
|
2495
|
+
// Nothing durable to reconcile against unless the queue was persisted.
|
|
2496
|
+
if (!this.deferredWritesPersisted && !this.quiesced) {
|
|
2497
|
+
this.unackedDeferredWrites = [];
|
|
2498
|
+
return;
|
|
2499
|
+
}
|
|
2500
|
+
try {
|
|
2501
|
+
this.store.sync();
|
|
2502
|
+
}
|
|
2503
|
+
catch (err) {
|
|
2504
|
+
console.error(`[host-mode] chronicle sync failed — ${this.unackedDeferredWrites.length} flushed deferred ` +
|
|
2505
|
+
`write(s) stay in the recovery queue until a sync succeeds:`, err);
|
|
2506
|
+
return;
|
|
2507
|
+
}
|
|
2508
|
+
this.unackedDeferredWrites = [];
|
|
2509
|
+
this.deferredScanFrom.clear();
|
|
2510
|
+
this.persistDeferredWrites();
|
|
2511
|
+
}
|
|
2512
|
+
/**
|
|
2513
|
+
* Move drained entries into the un-acked set and record, per target
|
|
2514
|
+
* agent, the store position they will be appended after — then persist
|
|
2515
|
+
* that receipt BEFORE any of them is written. Boot recovery scans from the
|
|
2516
|
+
* recorded position to the tail, so every member of an interrupted batch
|
|
2517
|
+
* is visible to the dedup, however large the batch.
|
|
2518
|
+
*/
|
|
2519
|
+
handOffDeferredWrites(entries) {
|
|
2520
|
+
if (entries.length === 0)
|
|
2521
|
+
return;
|
|
2522
|
+
const durable = this.deferredWritesPersisted || this.quiesced;
|
|
2523
|
+
if (!durable)
|
|
2524
|
+
return; // memory-only deferrals: nothing to reconcile at boot
|
|
2525
|
+
for (const entry of entries) {
|
|
2526
|
+
const target = entry.forAgent ?? this.primaryAgentName;
|
|
2527
|
+
const agent = target ? this.agents.get(target) : undefined;
|
|
2528
|
+
if (!agent)
|
|
2529
|
+
continue;
|
|
2530
|
+
let count = 0;
|
|
2531
|
+
try {
|
|
2532
|
+
const cm = agent.getContextManager();
|
|
2533
|
+
count = typeof cm.getMessageCount === 'function' ? cm.getMessageCount() : cm.getAllMessages().length;
|
|
2534
|
+
}
|
|
2535
|
+
catch {
|
|
2536
|
+
count = 0;
|
|
2537
|
+
}
|
|
2538
|
+
const prev = this.deferredScanFrom.get(target);
|
|
2539
|
+
this.deferredScanFrom.set(target, prev === undefined ? count : Math.min(prev, count));
|
|
2540
|
+
}
|
|
2541
|
+
this.unackedDeferredWrites.push(...entries);
|
|
2542
|
+
this.persistDeferredWrites();
|
|
2543
|
+
}
|
|
2544
|
+
/**
|
|
2545
|
+
* Which of `ids` already exist in some agent's store as a landed deferred
|
|
2546
|
+
* write (`metadata.deferredWriteId`). Scans each context manager's recent
|
|
2547
|
+
* tail blob-free — an interrupted flush is always within the last few
|
|
2548
|
+
* hundred slots — and tolerates facades without windowed reads.
|
|
2549
|
+
*/
|
|
2550
|
+
landedDeferredWriteIds(ids, scanFrom) {
|
|
2551
|
+
const landed = new Set();
|
|
2552
|
+
if (ids.size === 0)
|
|
2553
|
+
return landed;
|
|
2554
|
+
const WINDOW = 500;
|
|
2555
|
+
for (const agent of this.agents.values()) {
|
|
2556
|
+
try {
|
|
2557
|
+
const cm = agent.getContextManager();
|
|
2558
|
+
const consider = (m) => {
|
|
2559
|
+
const id = m.metadata?.deferredWriteId;
|
|
2560
|
+
if (typeof id === 'string' && ids.has(id))
|
|
2561
|
+
landed.add(id);
|
|
2562
|
+
};
|
|
2563
|
+
if (typeof cm.getMessageCount === 'function' && typeof cm.getMessageWindow === 'function') {
|
|
2564
|
+
const total = cm.getMessageCount();
|
|
2565
|
+
// From the receipt position (0 = whole store when unknown) to the
|
|
2566
|
+
// tail, in blob-free windows — the batch can only be after it.
|
|
2567
|
+
const from = Math.min(scanFrom.get(agent.name) ?? 0, total);
|
|
2568
|
+
for (let start = from; start < total; start += WINDOW) {
|
|
2569
|
+
const win = cm.getMessageWindow(start, Math.min(WINDOW, total - start), { resolveBlobs: false });
|
|
2570
|
+
for (const m of win.messages)
|
|
2571
|
+
consider(m);
|
|
2572
|
+
}
|
|
2573
|
+
}
|
|
2574
|
+
else {
|
|
2575
|
+
for (const m of cm.getAllMessages())
|
|
2576
|
+
consider(m);
|
|
2577
|
+
}
|
|
2578
|
+
}
|
|
2579
|
+
catch (err) {
|
|
2580
|
+
console.error(`[host-mode] could not scan ${agent.name} for landed deferred writes:`, err);
|
|
2581
|
+
}
|
|
2582
|
+
}
|
|
2583
|
+
return landed;
|
|
2584
|
+
}
|
|
2585
|
+
/** Atomic JSON write for the recovery files (tmp + rename, 0600) — the
|
|
2586
|
+
* same idiom as the Discord awareness outbox. */
|
|
2587
|
+
writeRecoveryFile(path, document) {
|
|
2588
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
2589
|
+
const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
|
|
2590
|
+
writeFileSync(temporary, `${JSON.stringify(document, null, 2)}\n`, { mode: 0o600 });
|
|
2591
|
+
renameSync(temporary, path);
|
|
2592
|
+
}
|
|
2593
|
+
readRecoveryFile(path) {
|
|
2594
|
+
if (!existsSync(path))
|
|
2595
|
+
return undefined;
|
|
2596
|
+
return JSON.parse(readFileSync(path, 'utf8'));
|
|
2597
|
+
}
|
|
2598
|
+
warnRecoveryFallbackOnce() {
|
|
2599
|
+
if (this.recoveryFallbackWarned)
|
|
2600
|
+
return;
|
|
2601
|
+
this.recoveryFallbackWarned = true;
|
|
2602
|
+
console.error('[host-mode] no storePath / hostModePath / deferredWritesPath: quiesce state and deferred ' +
|
|
2603
|
+
'writes persist in the branch-local framework/state slot — a historical rollback while ' +
|
|
2604
|
+
'quiesced can erase them; configure the recovery paths');
|
|
2605
|
+
}
|
|
2606
|
+
/**
|
|
2607
|
+
* Mirror the deferred-write queue durably while quiesced (and until an
|
|
2608
|
+
* already-persisted queue drains). The quiesce flag itself is persisted
|
|
2609
|
+
* because a crash mid-surgery is an expected event; the messages withheld
|
|
2610
|
+
* BECAUSE of that surgery would otherwise die with the process — and the
|
|
2611
|
+
* window is operator-length, not turn-length. Lives in the
|
|
2612
|
+
* branch-independent recovery file so a rollback cannot orphan it.
|
|
2613
|
+
* Deferrals outside a quiesce window (turn-alive, mid-tool-cycle) are
|
|
2614
|
+
* still memory-only: they flush within the turn, as before.
|
|
2615
|
+
*/
|
|
2616
|
+
persistDeferredWrites() {
|
|
2617
|
+
try {
|
|
2618
|
+
// The durable queue is everything not yet acked: writes still pending
|
|
2619
|
+
// AND writes handed to a context manager whose sync has not happened.
|
|
2620
|
+
// Always in original deferral order, whatever the pending/un-acked
|
|
2621
|
+
// split: a re-deferred entry must not jump the queue.
|
|
2622
|
+
const durable = [...this.unackedDeferredWrites, ...this.deferredMessages].sort(bySeq);
|
|
2623
|
+
const scanFrom = Object.fromEntries(this.deferredScanFrom);
|
|
2624
|
+
if ((this.quiesced || this.deferredWritesPersisted) && durable.length > 0) {
|
|
2625
|
+
const payload = JSON.stringify(durable);
|
|
2626
|
+
if (payload.length > DEFERRED_WRITES_PERSIST_CAP_BYTES) {
|
|
2627
|
+
if (!this.deferredWritesCapWarned) {
|
|
2628
|
+
this.deferredWritesCapWarned = true;
|
|
2629
|
+
console.error(`[host-mode] deferred context writes exceed ${DEFERRED_WRITES_PERSIST_CAP_BYTES} bytes ` +
|
|
2630
|
+
`serialized (${durable.length} message(s)) — kept in memory only; ` +
|
|
2631
|
+
`a crash before resume loses them`);
|
|
2632
|
+
}
|
|
2633
|
+
return;
|
|
2634
|
+
}
|
|
2635
|
+
if (this.deferredWritesPath) {
|
|
2636
|
+
this.writeRecoveryFile(this.deferredWritesPath, { version: 2, pending: durable, scanFrom });
|
|
2637
|
+
}
|
|
2638
|
+
else {
|
|
2639
|
+
this.warnRecoveryFallbackOnce();
|
|
2640
|
+
this.store.setStateJson(DEFERRED_WRITES_ID, { version: 2, pending: durable, scanFrom });
|
|
2641
|
+
}
|
|
2642
|
+
this.deferredWritesPersisted = true;
|
|
2643
|
+
}
|
|
2644
|
+
else if (this.deferredWritesPersisted) {
|
|
2645
|
+
if (this.deferredWritesPath) {
|
|
2646
|
+
this.writeRecoveryFile(this.deferredWritesPath, { version: 1, pending: [] });
|
|
2647
|
+
}
|
|
2648
|
+
else {
|
|
2649
|
+
this.store.setStateJson(DEFERRED_WRITES_ID, []);
|
|
2650
|
+
}
|
|
2651
|
+
this.deferredWritesPersisted = false;
|
|
2652
|
+
}
|
|
2653
|
+
}
|
|
2654
|
+
catch (err) {
|
|
2655
|
+
console.error('[host-mode] failed to persist deferred context writes:', err);
|
|
2656
|
+
}
|
|
2657
|
+
}
|
|
2658
|
+
restorePersistedDeferredWrites() {
|
|
2659
|
+
try {
|
|
2660
|
+
let raw;
|
|
2661
|
+
if (this.deferredWritesPath && existsSync(this.deferredWritesPath)) {
|
|
2662
|
+
raw = this.readRecoveryFile(this.deferredWritesPath);
|
|
2663
|
+
}
|
|
2664
|
+
else {
|
|
2665
|
+
// Stores that predate the recovery file (or store-only configs).
|
|
2666
|
+
raw = this.store.getStateJson(DEFERRED_WRITES_ID);
|
|
2667
|
+
}
|
|
2668
|
+
// v1 slot payloads were a bare array; v1/v2 files and v2 slots are
|
|
2669
|
+
// `{ pending, scanFrom? }`.
|
|
2670
|
+
const doc = Array.isArray(raw) ? { pending: raw } : raw;
|
|
2671
|
+
const data = doc?.pending;
|
|
2672
|
+
if (!Array.isArray(data) || data.length === 0)
|
|
2673
|
+
return 0;
|
|
2674
|
+
const scanFrom = new Map();
|
|
2675
|
+
if (doc?.scanFrom && typeof doc.scanFrom === 'object') {
|
|
2676
|
+
for (const [k, v] of Object.entries(doc.scanFrom)) {
|
|
2677
|
+
if (typeof v === 'number' && Number.isFinite(v))
|
|
2678
|
+
scanFrom.set(k, Math.max(0, Math.floor(v)));
|
|
2679
|
+
}
|
|
2680
|
+
}
|
|
2681
|
+
const candidates = data
|
|
2682
|
+
.filter((m) => !!m && typeof m === 'object' && typeof m.participant === 'string'
|
|
2683
|
+
&& Array.isArray(m.content))
|
|
2684
|
+
.map((m, i) => ({
|
|
2685
|
+
...m,
|
|
2686
|
+
id: typeof m.id === 'string' ? m.id : randomUUID(),
|
|
2687
|
+
// Files that predate `seq` are in write order already.
|
|
2688
|
+
seq: typeof m.seq === 'number' ? m.seq : i + 1,
|
|
2689
|
+
}))
|
|
2690
|
+
.sort(bySeq);
|
|
2691
|
+
this.deferredSeq = Math.max(this.deferredSeq, ...candidates.map((m) => m.seq));
|
|
2692
|
+
// Exactly-once: a message that landed before its ack was written is
|
|
2693
|
+
// already in the store under its deferredWriteId — do not replay it.
|
|
2694
|
+
// The scan covers every slot the interrupted batch could occupy (from
|
|
2695
|
+
// the receipt's per-agent position), or the whole store when the
|
|
2696
|
+
// receipt predates the position field.
|
|
2697
|
+
const landed = this.landedDeferredWriteIds(new Set(candidates.map((m) => m.id)), scanFrom);
|
|
2698
|
+
const restored = candidates.filter((m) => !landed.has(m.id));
|
|
2699
|
+
if (landed.size > 0) {
|
|
2700
|
+
console.error(`[host-mode] ${landed.size} deferred context write(s) had already landed before their ack ` +
|
|
2701
|
+
`(interrupted flush) — skipped, not replayed`);
|
|
2702
|
+
}
|
|
2703
|
+
this.deferredMessages.push(...restored);
|
|
2704
|
+
this.deferredWritesPersisted = true;
|
|
2705
|
+
// Rewrite the durable queue without the already-landed entries so a
|
|
2706
|
+
// second interrupted boot does not re-scan them.
|
|
2707
|
+
if (landed.size > 0)
|
|
2708
|
+
this.persistDeferredWrites();
|
|
2709
|
+
return restored.length;
|
|
2710
|
+
}
|
|
2711
|
+
catch (err) {
|
|
2712
|
+
console.error('[host-mode] failed to restore persisted deferred context writes:', err);
|
|
2713
|
+
return 0;
|
|
2714
|
+
}
|
|
2715
|
+
}
|
|
2716
|
+
/**
|
|
2717
|
+
* Operator-driven maintenance: run (or join) one bounded compression pass —
|
|
2718
|
+
* the same runQueuedMaintenance the timer drives, so tool definitions are
|
|
2719
|
+
* refreshed and ticks are bounded/serialized identically — and return the
|
|
2720
|
+
* maintenance snapshot. Works while quiesced by design: `running` stays
|
|
2721
|
+
* true in quiesce mode precisely so this machinery stays hot.
|
|
2722
|
+
*/
|
|
2723
|
+
async maintenanceTick() {
|
|
2724
|
+
if (!this.running) {
|
|
2725
|
+
// startQueuedMaintenance no-ops on a stopped/never-started framework —
|
|
2726
|
+
// a bare snapshot would masquerade as a completed pass, so say so.
|
|
2727
|
+
console.warn('[host-mode] maintenanceTick: framework is not running — no pass executed');
|
|
2728
|
+
return { ...this.getContextMaintenanceSnapshot(), ran: false };
|
|
2729
|
+
}
|
|
2730
|
+
this.startQueuedMaintenance();
|
|
2731
|
+
await this.maintenancePass;
|
|
2732
|
+
return { ...this.getContextMaintenanceSnapshot(), ran: true };
|
|
2733
|
+
}
|
|
1793
2734
|
/** Counts-only context-maintenance diagnostics for authenticated debug UIs. */
|
|
1794
2735
|
getContextMaintenanceSnapshot() {
|
|
1795
2736
|
const agents = [...this.agents.values()].map((agent) => {
|
|
@@ -1907,6 +2848,90 @@ export class AgentFramework {
|
|
|
1907
2848
|
// large store, and older context-manager builds ignore the argument.
|
|
1908
2849
|
return cm.previewContext({ maxTokens: budgetTokens, reserveForResponse: agent.maxTokens }, overrides, opts);
|
|
1909
2850
|
}
|
|
2851
|
+
/**
|
|
2852
|
+
* Feasibility preview for a runtime-settings patch — or, with no patch, for
|
|
2853
|
+
* the agent's CURRENT settings (the resume-gate case: quiesce maintenance
|
|
2854
|
+
* may have moved the folded floor and the branch generation, so the verdict
|
|
2855
|
+
* must be computed fresh at resume time).
|
|
2856
|
+
*
|
|
2857
|
+
* Models the patch through the same immediate-vs-paced semantics as the
|
|
2858
|
+
* apply path (`Agent.planRuntimeSettings`): a non-immediate budget decrease
|
|
2859
|
+
* is previewed at the UNCHANGED live compile budget, with a best-effort
|
|
2860
|
+
* advisory verdict at the descent target. This is what the superseded
|
|
2861
|
+
* `feat/budget-preflight-guard` got wrong — it previewed the patch value as
|
|
2862
|
+
* the compile budget and rejected safe-by-construction paced descents.
|
|
2863
|
+
*
|
|
2864
|
+
* Never throws for preview-layer reasons: strategy without previewContext,
|
|
2865
|
+
* preview overlap, stale branch generation, and the hierarchical (non-
|
|
2866
|
+
* adaptive) path all come back as `{available: false, reason}`. Unknown
|
|
2867
|
+
* agent still throws, matching the sibling accessors.
|
|
2868
|
+
*/
|
|
2869
|
+
previewAgentRuntimeSettings(agentName, patch) {
|
|
2870
|
+
const agent = this.agents.get(agentName);
|
|
2871
|
+
if (!agent)
|
|
2872
|
+
throw new Error(`Unknown agent: ${agentName}`);
|
|
2873
|
+
const plan = agent.planRuntimeSettings(patch);
|
|
2874
|
+
const settings = agent.getRuntimeSettings();
|
|
2875
|
+
const base = {
|
|
2876
|
+
available: false,
|
|
2877
|
+
path: plan.path,
|
|
2878
|
+
liveBudgetTokens: plan.liveBudgetTokens,
|
|
2879
|
+
transition: settings.transition,
|
|
2880
|
+
...(settings.transitionReason ? { transitionReason: settings.transitionReason } : {}),
|
|
2881
|
+
};
|
|
2882
|
+
const runPreview = (budgetTokens) => {
|
|
2883
|
+
try {
|
|
2884
|
+
const result = this.previewContextSettings(agentName, budgetTokens, Object.keys(plan.overrides).length > 0 ? plan.overrides : undefined);
|
|
2885
|
+
if (result === null || typeof result !== 'object') {
|
|
2886
|
+
return { reason: 'no_preview_support' };
|
|
2887
|
+
}
|
|
2888
|
+
return { result: result };
|
|
2889
|
+
}
|
|
2890
|
+
catch (error) {
|
|
2891
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2892
|
+
// Structured mapping of the strategy's known throw paths. Message
|
|
2893
|
+
// matching is fragile by nature, so unknown messages surface verbatim
|
|
2894
|
+
// as the reason rather than being swallowed or rethrown.
|
|
2895
|
+
if (message.includes('requires reinitialization for the current branch generation')) {
|
|
2896
|
+
return { reason: 'branch_generation_changed' };
|
|
2897
|
+
}
|
|
2898
|
+
if (message.includes('already running; previews must not overlap')) {
|
|
2899
|
+
return { reason: 'preview_in_flight' };
|
|
2900
|
+
}
|
|
2901
|
+
if (message.includes('requires adaptiveResolution')) {
|
|
2902
|
+
return { reason: 'no_adaptive_resolution' };
|
|
2903
|
+
}
|
|
2904
|
+
return { reason: message };
|
|
2905
|
+
}
|
|
2906
|
+
};
|
|
2907
|
+
const effective = runPreview(plan.effectiveBudgetTokens);
|
|
2908
|
+
if (!effective.result) {
|
|
2909
|
+
return { ...base, reason: effective.reason };
|
|
2910
|
+
}
|
|
2911
|
+
const preview = {
|
|
2912
|
+
...base,
|
|
2913
|
+
available: true,
|
|
2914
|
+
effective: {
|
|
2915
|
+
...effective.result,
|
|
2916
|
+
budgetTokens: plan.effectiveBudgetTokens,
|
|
2917
|
+
fits: effective.result.fits === true,
|
|
2918
|
+
finalTokens: Number(effective.result.finalTokens ?? 0),
|
|
2919
|
+
},
|
|
2920
|
+
};
|
|
2921
|
+
if (plan.advisoryTargetTokens !== undefined) {
|
|
2922
|
+
// Best-effort: an advisory failure never flips `available` — the
|
|
2923
|
+
// effective verdict above is the one that gates anything.
|
|
2924
|
+
const advisory = runPreview(plan.advisoryTargetTokens);
|
|
2925
|
+
if (advisory.result) {
|
|
2926
|
+
preview.advisory = {
|
|
2927
|
+
...advisory.result,
|
|
2928
|
+
targetTokens: plan.advisoryTargetTokens,
|
|
2929
|
+
fits: advisory.result.fits === true,
|
|
2930
|
+
};
|
|
2931
|
+
}
|
|
2932
|
+
}
|
|
2933
|
+
return preview;
|
|
2934
|
+
}
|
|
1910
2935
|
/**
|
|
1911
2936
|
* Check if process logging is enabled.
|
|
1912
2937
|
*/
|
|
@@ -2066,6 +3091,81 @@ export class AgentFramework {
|
|
|
2066
3091
|
state.agentRuntimeSettings = all;
|
|
2067
3092
|
this.store.setStateJson(FRAMEWORK_STATE_ID, state);
|
|
2068
3093
|
}
|
|
3094
|
+
/** Persist (or clear, with null) the host quiesce mode. First process-global
|
|
3095
|
+
* key in the framework/state slot: quiesce survives a restart BY DESIGN — a
|
|
3096
|
+
* crash mid-surgery must come back up NOT serving against a half-repaired
|
|
3097
|
+
* store. Resume is always reachable (public method, WS/HTTP, control-plane
|
|
3098
|
+
* host/command), and every quiesced boot logs a loud banner. */
|
|
3099
|
+
persistHostMode(mode) {
|
|
3100
|
+
if (this.hostModePath) {
|
|
3101
|
+
// Branch-independent: the recovery file is the record. (`quiesced:
|
|
3102
|
+
// false` is written explicitly rather than deleting the file so that
|
|
3103
|
+
// its presence, not its content, decides precedence over the legacy
|
|
3104
|
+
// branch-local slot below.)
|
|
3105
|
+
this.writeRecoveryFile(this.hostModePath, mode === null ? { version: 1, quiesced: false, clearedAt: Date.now() } : { version: 1, ...mode });
|
|
3106
|
+
if (mode === null) {
|
|
3107
|
+
// Also drop any legacy slot marker so a downgrade cannot resurrect it.
|
|
3108
|
+
try {
|
|
3109
|
+
const data = this.store.getStateJson(FRAMEWORK_STATE_ID);
|
|
3110
|
+
if (data && typeof data === 'object' && 'hostMode' in data) {
|
|
3111
|
+
const state = { ...data };
|
|
3112
|
+
delete state.hostMode;
|
|
3113
|
+
this.store.setStateJson(FRAMEWORK_STATE_ID, state);
|
|
3114
|
+
}
|
|
3115
|
+
}
|
|
3116
|
+
catch { /* best effort */ }
|
|
3117
|
+
}
|
|
3118
|
+
}
|
|
3119
|
+
else {
|
|
3120
|
+
this.warnRecoveryFallbackOnce();
|
|
3121
|
+
const data = this.store.getStateJson(FRAMEWORK_STATE_ID);
|
|
3122
|
+
const state = (data && typeof data === 'object' ? data : {});
|
|
3123
|
+
if (mode === null)
|
|
3124
|
+
delete state.hostMode;
|
|
3125
|
+
else
|
|
3126
|
+
state.hostMode = { ...mode };
|
|
3127
|
+
this.store.setStateJson(FRAMEWORK_STATE_ID, state);
|
|
3128
|
+
}
|
|
3129
|
+
// The whole feature rests on the flag surviving a crash — close the
|
|
3130
|
+
// window between the write and the next periodic sync.
|
|
3131
|
+
try {
|
|
3132
|
+
this.store.sync();
|
|
3133
|
+
}
|
|
3134
|
+
catch (err) {
|
|
3135
|
+
console.error('[host-mode] store sync after host-mode change failed:', err);
|
|
3136
|
+
}
|
|
3137
|
+
}
|
|
3138
|
+
/** The recovery file wins whenever it exists; the branch-local slot is
|
|
3139
|
+
* read only for stores that predate it (or store-only configs). */
|
|
3140
|
+
readHostMode() {
|
|
3141
|
+
try {
|
|
3142
|
+
let record = null;
|
|
3143
|
+
if (this.hostModePath && existsSync(this.hostModePath)) {
|
|
3144
|
+
const doc = this.readRecoveryFile(this.hostModePath);
|
|
3145
|
+
if (doc && typeof doc === 'object')
|
|
3146
|
+
record = doc;
|
|
3147
|
+
}
|
|
3148
|
+
else {
|
|
3149
|
+
const data = this.store.getStateJson(FRAMEWORK_STATE_ID);
|
|
3150
|
+
if (!data || typeof data !== 'object')
|
|
3151
|
+
return null;
|
|
3152
|
+
const mode = data.hostMode;
|
|
3153
|
+
if (!mode || typeof mode !== 'object')
|
|
3154
|
+
return null;
|
|
3155
|
+
record = mode;
|
|
3156
|
+
}
|
|
3157
|
+
if (!record)
|
|
3158
|
+
return null;
|
|
3159
|
+
return {
|
|
3160
|
+
quiesced: record.quiesced === true,
|
|
3161
|
+
...(typeof record.reason === 'string' ? { reason: record.reason } : {}),
|
|
3162
|
+
...(typeof record.since === 'number' ? { since: record.since } : {}),
|
|
3163
|
+
};
|
|
3164
|
+
}
|
|
3165
|
+
catch {
|
|
3166
|
+
return null;
|
|
3167
|
+
}
|
|
3168
|
+
}
|
|
2069
3169
|
/**
|
|
2070
3170
|
* Create an ephemeral agent that is NOT registered in the main event loop.
|
|
2071
3171
|
*
|
|
@@ -2176,18 +3276,36 @@ export class AgentFramework {
|
|
|
2176
3276
|
* The task message should already be in the context manager.
|
|
2177
3277
|
*/
|
|
2178
3278
|
async runEphemeralToCompletion(agent, contextManager, watchdogs) {
|
|
3279
|
+
// Refused at admission while quiesced rather than requeued: the caller
|
|
3280
|
+
// awaits the settle promise under an idle-stall watchdog, so a gate-parked
|
|
3281
|
+
// ephemeral would ride out its timeout and reject with a misleading
|
|
3282
|
+
// "stalled" error hours later. A clear refusal now beats that. Checked
|
|
3283
|
+
// before the generation ticket is consumed so the same (agent, cm) pair
|
|
3284
|
+
// can be retried after resume().
|
|
3285
|
+
if (this.quiesced) {
|
|
3286
|
+
throw new Error(`framework is quiesced${this.quiesceReason ? ` (${this.quiesceReason})` : ''}: ` +
|
|
3287
|
+
`refusing new ephemeral run for ${agent.name} — resume() first`);
|
|
3288
|
+
}
|
|
2179
3289
|
// Only a fresh object returned by createEphemeralAgent may enter this path.
|
|
2180
3290
|
// Never overwrite a resident/conversation owner or a concurrent run: their
|
|
2181
3291
|
// cleanup is name-keyed and could cancel/deregister the legitimate owner.
|
|
2182
3292
|
if (this.ephemeralCandidates.get(agent) !== contextManager) {
|
|
2183
3293
|
throw new Error(`Ephemeral agent "${agent.name}" has no fresh generation ticket from this framework`);
|
|
2184
3294
|
}
|
|
3295
|
+
// A live surgery holds the whole store: an agent admitted now would
|
|
3296
|
+
// compile against a history that is being replaced under it. Refuse
|
|
3297
|
+
// BEFORE consuming the ticket so the caller can retry once it completes.
|
|
3298
|
+
if (this.surgeryHold) {
|
|
3299
|
+
throw new Error(`Ephemeral agent "${agent.name}" refused: the store is under live ${this.surgeryHold.verb} ` +
|
|
3300
|
+
`for ${this.surgeryHold.agentName} — retry when it completes`);
|
|
3301
|
+
}
|
|
2185
3302
|
// Consume before any await/registration: the exact (Agent, ContextManager)
|
|
2186
3303
|
// generation is one-shot even if startup later fails.
|
|
2187
3304
|
this.ephemeralCandidates.delete(agent);
|
|
2188
3305
|
if (this.agents.has(agent.name) || this.ephemeralRuns.has(agent.name)) {
|
|
2189
3306
|
throw new Error(`Ephemeral agent "${agent.name}" is already registered or running`);
|
|
2190
3307
|
}
|
|
3308
|
+
// Register temporarily so the event loop can drive it
|
|
2191
3309
|
this.agents.set(agent.name, agent);
|
|
2192
3310
|
const run = {
|
|
2193
3311
|
settle: this.createDeferred(),
|
|
@@ -2890,8 +4008,15 @@ export class AgentFramework {
|
|
|
2890
4008
|
source: 'admin',
|
|
2891
4009
|
timestamp: Date.now(),
|
|
2892
4010
|
});
|
|
2893
|
-
console.error(`[nudge] agent=${name} queued by=${requestedBy ?? 'unknown'} status=${agentStatus}`
|
|
2894
|
-
|
|
4011
|
+
console.error(`[nudge] agent=${name} queued by=${requestedBy ?? 'unknown'} status=${agentStatus}` +
|
|
4012
|
+
(this.quiesced ? ' (host quiesced — parked until resume)' : ''));
|
|
4013
|
+
this.recordOperatorAction({
|
|
4014
|
+
kind: 'nudge',
|
|
4015
|
+
agent: name,
|
|
4016
|
+
requester: { via: 'nudge', ...(requestedBy ? { name: requestedBy } : {}) },
|
|
4017
|
+
result: { agentStatus, ...(this.quiesced ? { quiesced: true } : {}) },
|
|
4018
|
+
});
|
|
4019
|
+
return { ok: true, agentName: name, agentStatus, ...(this.quiesced ? { quiesced: true } : {}) };
|
|
2895
4020
|
}
|
|
2896
4021
|
/**
|
|
2897
4022
|
* Handle a `host/command` request from an MCPL surface server (e.g. a
|
|
@@ -2904,14 +4029,65 @@ export class AgentFramework {
|
|
|
2904
4029
|
*
|
|
2905
4030
|
* nudge — run inference on the current context with NO new events
|
|
2906
4031
|
* (see `nudgeAgent`).
|
|
4032
|
+
*
|
|
4033
|
+
* quiesce — enter host maintenance mode (issue #122): park wakes, pause
|
|
4034
|
+
* MCPL data planes, drain in-flight turns (optionally `abandon` on
|
|
4035
|
+
* timeout). Host-scoped, no agentName.
|
|
4036
|
+
*
|
|
4037
|
+
* resume — leave maintenance mode behind a fresh feasibility preview of
|
|
4038
|
+
* every agent's current settings (`force` overrides a failing verdict).
|
|
4039
|
+
*
|
|
4040
|
+
* maintain — run (or join) one bounded compression/maintenance pass and
|
|
4041
|
+
* return the snapshot; the operator lever for draining quarantine or
|
|
4042
|
+
* advancing merges during a quiesce window.
|
|
4043
|
+
*
|
|
4044
|
+
* host-status — report the quiesce state (drained / parked wakes /
|
|
4045
|
+
* running background scripts).
|
|
2907
4046
|
*/
|
|
2908
4047
|
async handleHostCommand(serverId, params) {
|
|
2909
4048
|
if (params.command !== 'undo' &&
|
|
2910
4049
|
params.command !== 'hide' &&
|
|
2911
4050
|
params.command !== 'unstick' &&
|
|
2912
|
-
params.command !== 'nudge'
|
|
4051
|
+
params.command !== 'nudge' &&
|
|
4052
|
+
params.command !== 'quiesce' &&
|
|
4053
|
+
params.command !== 'resume' &&
|
|
4054
|
+
params.command !== 'maintain' &&
|
|
4055
|
+
params.command !== 'host-status') {
|
|
2913
4056
|
return { ok: false, error: `Unknown host command: ${String(params.command)}` };
|
|
2914
4057
|
}
|
|
4058
|
+
// Host-scoped verbs (issue #122) — no agent resolution.
|
|
4059
|
+
const requester = params.requesterName ?? params.requesterId ?? `mcpl:${serverId}`;
|
|
4060
|
+
if (params.command === 'quiesce') {
|
|
4061
|
+
// quiesce() clamps timeoutMs to [1s, 10m] itself — one clamp for every ingress.
|
|
4062
|
+
console.error(`[host-command] quiesce by=${requester} (server=${serverId})`);
|
|
4063
|
+
const hostMode = await this.quiesce({
|
|
4064
|
+
reason: params.reason ?? `host/command by ${requester}`,
|
|
4065
|
+
...(params.timeoutMs !== undefined ? { timeoutMs: params.timeoutMs } : {}),
|
|
4066
|
+
...(params.abandon ? { abandon: true } : {}),
|
|
4067
|
+
});
|
|
4068
|
+
return { ok: true, hostMode };
|
|
4069
|
+
}
|
|
4070
|
+
if (params.command === 'resume') {
|
|
4071
|
+
console.error(`[host-command] resume by=${requester} (server=${serverId})`);
|
|
4072
|
+
try {
|
|
4073
|
+
const hostMode = await this.resume(params.force ? { force: true } : undefined);
|
|
4074
|
+
return { ok: true, hostMode };
|
|
4075
|
+
}
|
|
4076
|
+
catch (error) {
|
|
4077
|
+
if (error instanceof ResumeBlockedError) {
|
|
4078
|
+
return { ok: false, error: error.message, hostMode: this.getHostModeStatus() };
|
|
4079
|
+
}
|
|
4080
|
+
throw error;
|
|
4081
|
+
}
|
|
4082
|
+
}
|
|
4083
|
+
if (params.command === 'maintain') {
|
|
4084
|
+
console.error(`[host-command] maintain by=${requester} (server=${serverId})`);
|
|
4085
|
+
const maintenance = await this.maintenanceTick();
|
|
4086
|
+
return { ok: true, maintenance, hostMode: this.getHostModeStatus() };
|
|
4087
|
+
}
|
|
4088
|
+
if (params.command === 'host-status') {
|
|
4089
|
+
return { ok: true, hostMode: this.getHostModeStatus() };
|
|
4090
|
+
}
|
|
2915
4091
|
const agentName = params.agentName ?? [...this.agents.keys()][0];
|
|
2916
4092
|
if (!agentName || !this.agents.has(agentName)) {
|
|
2917
4093
|
return { ok: false, error: `Unknown agent: ${String(agentName)}` };
|
|
@@ -2920,7 +4096,7 @@ export class AgentFramework {
|
|
|
2920
4096
|
// event, no context mutation (see nudgeAgent).
|
|
2921
4097
|
if (params.command === 'nudge') {
|
|
2922
4098
|
const r = this.nudgeAgent(agentName, params.requesterName ?? params.requesterId ?? `mcpl:${serverId}`);
|
|
2923
|
-
return { ok: r.ok, error: r.error, agentStatus: r.agentStatus };
|
|
4099
|
+
return { ok: r.ok, error: r.error, agentStatus: r.agentStatus, ...(r.quiesced ? { quiesced: true } : {}) };
|
|
2924
4100
|
}
|
|
2925
4101
|
// unstick: force the refusal-rewind loop on demand (even if the agent's
|
|
2926
4102
|
// autoRewind toggle is off). Redacts the turn that fed the refusal and
|
|
@@ -2948,7 +4124,14 @@ export class AgentFramework {
|
|
|
2948
4124
|
});
|
|
2949
4125
|
console.error(`[unstick] agent=${agentName} started cap=${cap} ` +
|
|
2950
4126
|
`by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`);
|
|
2951
|
-
|
|
4127
|
+
this.recordOperatorAction({
|
|
4128
|
+
kind: 'unstick',
|
|
4129
|
+
agent: agentName,
|
|
4130
|
+
requester: hostCommandRequester(serverId, params),
|
|
4131
|
+
params: { cap },
|
|
4132
|
+
result: { started: true, ...(this.quiesced ? { quiesced: true } : {}) },
|
|
4133
|
+
});
|
|
4134
|
+
return { ok: true, started: true, cap, ...(this.quiesced ? { quiesced: true } : {}) };
|
|
2952
4135
|
}
|
|
2953
4136
|
// hide: redact a single message (or an inclusive range) from the active
|
|
2954
4137
|
// branch, addressed by Discord message id. Unlike undo this is a
|
|
@@ -2993,9 +4176,23 @@ export class AgentFramework {
|
|
|
2993
4176
|
}
|
|
2994
4177
|
const [lo, hi] = fromIdx <= toIdx ? [fromIdx, toIdx] : [toIdx, fromIdx];
|
|
2995
4178
|
const refs = refsIn(lo, hi);
|
|
4179
|
+
const rangeIds = all.slice(lo, hi + 1).map((m) => String(m.id));
|
|
2996
4180
|
cm.removeMessages(all[lo].id, all[hi].id);
|
|
2997
4181
|
console.error(`[host-command] hide agent=${agentName} range removed=${hi - lo + 1} ` +
|
|
2998
4182
|
`(${params.fromMessageId}..${params.toMessageId}) by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`);
|
|
4183
|
+
const loggedRange = capIds(rangeIds);
|
|
4184
|
+
this.recordOperatorAction({
|
|
4185
|
+
kind: 'hide',
|
|
4186
|
+
agent: agentName,
|
|
4187
|
+
requester: hostCommandRequester(serverId, params),
|
|
4188
|
+
params: { fromMessageId: params.fromMessageId, toMessageId: params.toMessageId },
|
|
4189
|
+
result: {
|
|
4190
|
+
branch: this.store.currentBranch().name,
|
|
4191
|
+
hidden: hi - lo + 1,
|
|
4192
|
+
removedIds: loggedRange.ids,
|
|
4193
|
+
...(loggedRange.truncated ? { removedIdsTruncated: true } : {}),
|
|
4194
|
+
},
|
|
4195
|
+
});
|
|
2999
4196
|
return {
|
|
3000
4197
|
ok: true,
|
|
3001
4198
|
hidden: hi - lo + 1,
|
|
@@ -3004,9 +4201,17 @@ export class AgentFramework {
|
|
|
3004
4201
|
};
|
|
3005
4202
|
}
|
|
3006
4203
|
const refs = refsIn(fromIdx, fromIdx);
|
|
4204
|
+
const hiddenId = String(all[fromIdx].id);
|
|
3007
4205
|
cm.removeMessage(all[fromIdx].id);
|
|
3008
4206
|
console.error(`[host-command] hide agent=${agentName} removed=1 (${params.fromMessageId}) ` +
|
|
3009
4207
|
`by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`);
|
|
4208
|
+
this.recordOperatorAction({
|
|
4209
|
+
kind: 'hide',
|
|
4210
|
+
agent: agentName,
|
|
4211
|
+
requester: hostCommandRequester(serverId, params),
|
|
4212
|
+
params: { fromMessageId: params.fromMessageId },
|
|
4213
|
+
result: { branch: this.store.currentBranch().name, hidden: 1, removedIds: [hiddenId] },
|
|
4214
|
+
});
|
|
3010
4215
|
return {
|
|
3011
4216
|
ok: true,
|
|
3012
4217
|
hidden: 1,
|
|
@@ -3029,55 +4234,40 @@ export class AgentFramework {
|
|
|
3029
4234
|
}
|
|
3030
4235
|
const n = Math.max(1, Math.min(50, Math.floor(params.messages)));
|
|
3031
4236
|
const cm = agent.getContextManager();
|
|
3032
|
-
const
|
|
3033
|
-
if (n >=
|
|
4237
|
+
const total = cm.getMessageCount();
|
|
4238
|
+
if (n >= total) {
|
|
3034
4239
|
return {
|
|
3035
4240
|
ok: false,
|
|
3036
|
-
error: `Cannot remove ${n} message(s) — history has ${
|
|
4241
|
+
error: `Cannot remove ${n} message(s) — history has ${total}; at least one must remain.`,
|
|
4242
|
+
};
|
|
4243
|
+
}
|
|
4244
|
+
const target = cm.getMessageWindow(total - 1 - n, 1, { resolveBlobs: false }).messages[0];
|
|
4245
|
+
try {
|
|
4246
|
+
const r = await this.rollbackToMessage(agentName, {
|
|
4247
|
+
messageId: String(target.id),
|
|
4248
|
+
branchName: `undo-msgs/${agentName}/${Date.now()}`,
|
|
4249
|
+
requester: hostCommandRequester(serverId, params),
|
|
4250
|
+
});
|
|
4251
|
+
return {
|
|
4252
|
+
ok: true,
|
|
4253
|
+
messagesRemoved: r.messagesRemoved,
|
|
4254
|
+
removedRefs: r.removedRefs,
|
|
4255
|
+
lastVisible: r.lastVisible,
|
|
4256
|
+
};
|
|
4257
|
+
}
|
|
4258
|
+
catch (error) {
|
|
4259
|
+
return {
|
|
4260
|
+
ok: false,
|
|
4261
|
+
error: error instanceof Error ? error.message : String(error),
|
|
4262
|
+
...(error instanceof OperatorActionError ? { code: error.code } : {}),
|
|
3037
4263
|
};
|
|
3038
4264
|
}
|
|
3039
|
-
const target = allMessages[allMessages.length - 1 - n];
|
|
3040
|
-
const discarded = allMessages.slice(allMessages.length - n);
|
|
3041
|
-
const removedRefs = extractDiscordAwarenessRefs(discarded);
|
|
3042
|
-
const sourceBranch = this.store.currentBranch().name;
|
|
3043
|
-
const targetBranch = `undo-msgs/${agentName}/${Date.now()}`;
|
|
3044
|
-
// Prepare the external side effect before switching Chronicle. If the
|
|
3045
|
-
// process dies after the switch but before activate(), startup promotes
|
|
3046
|
-
// this batch by matching targetBranch to the active branch.
|
|
3047
|
-
const markerBatch = this.discordAwarenessOutbox?.prepare({
|
|
3048
|
-
agentName,
|
|
3049
|
-
sourceBranch,
|
|
3050
|
-
targetBranch,
|
|
3051
|
-
refs: removedRefs,
|
|
3052
|
-
emoji: this.discordAwarenessEmoji,
|
|
3053
|
-
}) ?? null;
|
|
3054
|
-
const branchName = cm.branchAt(target.id, targetBranch);
|
|
3055
|
-
await cm.switchBranch(branchName);
|
|
3056
|
-
if (markerBatch)
|
|
3057
|
-
this.discordAwarenessOutbox.activate(markerBatch.id);
|
|
3058
|
-
await this.syncDiscordAwarenessMarkers();
|
|
3059
|
-
// Materialize config files from the new branch (fire-and-forget; gate
|
|
3060
|
-
// picks up via mtime) — mirrors undoLastTurn.
|
|
3061
|
-
const wsUndo = this.moduleRegistry.getModule('workspace');
|
|
3062
|
-
if (wsUndo && 'materializeMount' in wsUndo) {
|
|
3063
|
-
wsUndo
|
|
3064
|
-
.materializeMount('_config')
|
|
3065
|
-
.catch(() => { });
|
|
3066
|
-
}
|
|
3067
|
-
console.error(`[host-command] undo-messages agent=${agentName} removed=${n} branch=${branchName}` +
|
|
3068
|
-
` by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`);
|
|
3069
|
-
return {
|
|
3070
|
-
ok: true,
|
|
3071
|
-
messagesRemoved: n,
|
|
3072
|
-
removedRefs,
|
|
3073
|
-
lastVisible: await this.lastVisiblePreview(agentName),
|
|
3074
|
-
};
|
|
3075
4265
|
}
|
|
3076
4266
|
const requested = Math.max(1, Math.min(20, Math.floor(params.turns ?? 1)));
|
|
3077
4267
|
let undone = 0;
|
|
3078
4268
|
try {
|
|
3079
4269
|
for (let i = 0; i < requested; i++) {
|
|
3080
|
-
const r = this.undoLastTurn(agentName);
|
|
4270
|
+
const r = this.undoLastTurn(agentName, hostCommandRequester(serverId, params));
|
|
3081
4271
|
if (!r.undone)
|
|
3082
4272
|
break;
|
|
3083
4273
|
undone++;
|
|
@@ -3367,7 +4557,7 @@ export class AgentFramework {
|
|
|
3367
4557
|
* The undone branch is saved so `redo()` can restore it.
|
|
3368
4558
|
* Returns the checkpoint that was undone, or null if nothing to undo.
|
|
3369
4559
|
*/
|
|
3370
|
-
undoLastTurn(agentName) {
|
|
4560
|
+
undoLastTurn(agentName, requester) {
|
|
3371
4561
|
const agent = this.agents.get(agentName);
|
|
3372
4562
|
if (!agent) {
|
|
3373
4563
|
throw new Error(`Unknown agent: ${agentName}`);
|
|
@@ -3404,6 +4594,13 @@ export class AgentFramework {
|
|
|
3404
4594
|
fromBranch: currentBranch.name,
|
|
3405
4595
|
toBranch: undoBranchName,
|
|
3406
4596
|
});
|
|
4597
|
+
this.recordOperatorAction({
|
|
4598
|
+
kind: 'undo-turn',
|
|
4599
|
+
agent: agentName,
|
|
4600
|
+
...(requester ? { requester } : {}),
|
|
4601
|
+
params: { turnIndex: checkpoint.turnIndex },
|
|
4602
|
+
result: { sourceBranch: currentBranch.name, targetBranch: undoBranchName },
|
|
4603
|
+
});
|
|
3407
4604
|
return {
|
|
3408
4605
|
undone: true,
|
|
3409
4606
|
turnIndex: checkpoint.turnIndex,
|
|
@@ -3417,7 +4614,7 @@ export class AgentFramework {
|
|
|
3417
4614
|
* Switches back to the branch that was active before the last undo.
|
|
3418
4615
|
* Returns false if there's nothing to redo.
|
|
3419
4616
|
*/
|
|
3420
|
-
redo(agentName) {
|
|
4617
|
+
redo(agentName, requester) {
|
|
3421
4618
|
const agent = this.agents.get(agentName);
|
|
3422
4619
|
if (!agent) {
|
|
3423
4620
|
throw new Error(`Unknown agent: ${agentName}`);
|
|
@@ -3447,12 +4644,424 @@ export class AgentFramework {
|
|
|
3447
4644
|
fromBranch: currentBranch.name,
|
|
3448
4645
|
toBranch: branchName,
|
|
3449
4646
|
});
|
|
4647
|
+
this.recordOperatorAction({
|
|
4648
|
+
kind: 'redo-turn',
|
|
4649
|
+
agent: agentName,
|
|
4650
|
+
...(requester ? { requester } : {}),
|
|
4651
|
+
result: { sourceBranch: currentBranch.name, targetBranch: branchName },
|
|
4652
|
+
});
|
|
3450
4653
|
return {
|
|
3451
4654
|
redone: true,
|
|
3452
4655
|
fromBranch: currentBranch.name,
|
|
3453
4656
|
toBranch: branchName,
|
|
3454
4657
|
};
|
|
3455
4658
|
}
|
|
4659
|
+
// ---------------------------------------------------------------------------
|
|
4660
|
+
// Live operator surgery — rollback / suppress on the OPEN store, without a
|
|
4661
|
+
// restart — plus the durable action log every operator mutation writes to.
|
|
4662
|
+
//
|
|
4663
|
+
// Both follow the same idiom the offline surgeries use (recovery/
|
|
4664
|
+
// offline-branch.ts): fork first, mutate the fork, and make the fork the
|
|
4665
|
+
// active branch. The parent branch keeps everything, so restore is a
|
|
4666
|
+
// checkout. Mutations are refused (never queued) while the agent is not
|
|
4667
|
+
// idle — quiesce the host first if it is busy.
|
|
4668
|
+
// ---------------------------------------------------------------------------
|
|
4669
|
+
/**
|
|
4670
|
+
* Append a record to the operator action log and broadcast it as an
|
|
4671
|
+
* `operator:action` trace. Hosts use this directly for actions the
|
|
4672
|
+
* framework does not own (e.g. a WebUI toggling quiesce). Never throws.
|
|
4673
|
+
*/
|
|
4674
|
+
recordOperatorAction(input) {
|
|
4675
|
+
// Observability never fails the action: OperatorLog.append never throws,
|
|
4676
|
+
// and a misbehaving trace listener is contained here.
|
|
4677
|
+
const entry = this.operatorLog.append(input);
|
|
4678
|
+
try {
|
|
4679
|
+
this.emitTrace({
|
|
4680
|
+
type: 'operator:action',
|
|
4681
|
+
kind: entry.kind,
|
|
4682
|
+
...(entry.agent !== undefined ? { agentName: entry.agent } : {}),
|
|
4683
|
+
...(entry.requester !== undefined ? { requester: entry.requester } : {}),
|
|
4684
|
+
...(entry.note !== undefined ? { note: entry.note } : {}),
|
|
4685
|
+
...(entry.params !== undefined ? { params: entry.params } : {}),
|
|
4686
|
+
...(entry.result !== undefined ? { result: entry.result } : {}),
|
|
4687
|
+
...(entry.error !== undefined ? { error: entry.error } : {}),
|
|
4688
|
+
});
|
|
4689
|
+
}
|
|
4690
|
+
catch {
|
|
4691
|
+
// swallowed by design
|
|
4692
|
+
}
|
|
4693
|
+
return entry;
|
|
4694
|
+
}
|
|
4695
|
+
/** Newest `limit` operator-log entries, oldest first. */
|
|
4696
|
+
getOperatorLog(opts) {
|
|
4697
|
+
return this.operatorLog.readTail(opts?.limit ?? 100);
|
|
4698
|
+
}
|
|
4699
|
+
/** Where the operator log is written, or undefined when disabled. */
|
|
4700
|
+
getOperatorLogPath() {
|
|
4701
|
+
return this.operatorLog.path;
|
|
4702
|
+
}
|
|
4703
|
+
/**
|
|
4704
|
+
* The surgery gate — STORE-wide, not per agent. A branch switch moves the
|
|
4705
|
+
* active chronicle branch for every context manager sharing this store,
|
|
4706
|
+
* so an in-flight turn on *any* agent would have its next writes land on
|
|
4707
|
+
* a history its request was never compiled against. Every agent must be
|
|
4708
|
+
* idle with no turn alive (`state.status === 'idle'` alone is not enough:
|
|
4709
|
+
* status reads idle from dequeue until the stream registers and again
|
|
4710
|
+
* during teardown — the scheduler's own busy test is status + turn-alive,
|
|
4711
|
+
* the predicate `puppetToolCall` uses).
|
|
4712
|
+
*
|
|
4713
|
+
* Passing the gate RESERVES the store for the caller: a turn token is
|
|
4714
|
+
* held for every agent until the returned release runs, so no wake — via
|
|
4715
|
+
* the scheduler or a parked provider admission, both of which re-test
|
|
4716
|
+
* turn-alive — can start a turn while the switch is awaited. Release also
|
|
4717
|
+
* flushes messages that deferred behind the reservation, as the puppet
|
|
4718
|
+
* does. Always release in `finally`: a token nobody clears wedges the fleet.
|
|
4719
|
+
*/
|
|
4720
|
+
reserveStoreForSurgery(verb, agentName) {
|
|
4721
|
+
if (this.surgeryHold) {
|
|
4722
|
+
throw new OperatorActionError('agent-busy', `Cannot ${verb}: a live ${this.surgeryHold.verb} for ${this.surgeryHold.agentName} is already in progress`);
|
|
4723
|
+
}
|
|
4724
|
+
const busy = [];
|
|
4725
|
+
for (const [name, a] of this.agents) {
|
|
4726
|
+
if (a.state.status !== 'idle')
|
|
4727
|
+
busy.push(`${name} is ${a.state.status}`);
|
|
4728
|
+
else if (this.activeTurnTokens.has(name))
|
|
4729
|
+
busy.push(`${name} is idle+turn-alive`);
|
|
4730
|
+
}
|
|
4731
|
+
for (const name of this.activeTurnTokens.keys()) {
|
|
4732
|
+
if (!this.agents.has(name))
|
|
4733
|
+
busy.push(`${name} is turn-alive`);
|
|
4734
|
+
}
|
|
4735
|
+
if (busy.length > 0) {
|
|
4736
|
+
throw new OperatorActionError('agent-busy', `Cannot ${verb} while ${busy.join(', ')} — every agent sharing the store must be idle (quiesce the host first)`);
|
|
4737
|
+
}
|
|
4738
|
+
const reserved = new Map();
|
|
4739
|
+
for (const name of this.agents.keys()) {
|
|
4740
|
+
const token = this.nextTurnToken++;
|
|
4741
|
+
this.activeTurnTokens.set(name, token);
|
|
4742
|
+
reserved.set(name, token);
|
|
4743
|
+
}
|
|
4744
|
+
// The token snapshot only covers agents that exist now; the hold covers
|
|
4745
|
+
// arrivals during the awaited switch (see surgeryHold).
|
|
4746
|
+
const hold = { verb, agentName, since: Date.now() };
|
|
4747
|
+
this.surgeryHold = hold;
|
|
4748
|
+
let released = false;
|
|
4749
|
+
return () => {
|
|
4750
|
+
if (released)
|
|
4751
|
+
return;
|
|
4752
|
+
released = true;
|
|
4753
|
+
if (this.surgeryHold === hold)
|
|
4754
|
+
this.surgeryHold = null;
|
|
4755
|
+
for (const [name, token] of reserved) {
|
|
4756
|
+
if (this.activeTurnTokens.get(name) === token)
|
|
4757
|
+
this.activeTurnTokens.delete(name);
|
|
4758
|
+
}
|
|
4759
|
+
// Writers that deferred behind the reservation land now, on the branch
|
|
4760
|
+
// the operator chose (or the restored source) — the same end-of-turn
|
|
4761
|
+
// flush driveStream/puppet perform, under the same guard.
|
|
4762
|
+
if (this.deferredMessages.length > 0 && this.pendingAssistantBlocks.size === 0) {
|
|
4763
|
+
for (const name of reserved.keys()) {
|
|
4764
|
+
if (this.activeTurnTokens.has(name))
|
|
4765
|
+
continue;
|
|
4766
|
+
for (const msg of this.drainDeferredFor(name)) {
|
|
4767
|
+
try {
|
|
4768
|
+
this.addMessage(msg.participant, msg.content, msg.metadata, msg.forAgent ? { forAgent: msg.forAgent } : undefined);
|
|
4769
|
+
}
|
|
4770
|
+
catch (error) {
|
|
4771
|
+
console.error(`[operator] post-surgery deferred flush failed for ${name}: ${error instanceof Error ? error.message : String(error)}`);
|
|
4772
|
+
}
|
|
4773
|
+
}
|
|
4774
|
+
}
|
|
4775
|
+
}
|
|
4776
|
+
};
|
|
4777
|
+
}
|
|
4778
|
+
/**
|
|
4779
|
+
* After a failed switch or redaction: put the store back on `sourceBranch`
|
|
4780
|
+
* if it moved, and return a sentence saying what actually happened. Never
|
|
4781
|
+
* throws — the caller reports the original failure with this appended.
|
|
4782
|
+
*/
|
|
4783
|
+
async restoreSourceBranch(cm, sourceBranch, failedBranch, agentName) {
|
|
4784
|
+
const current = this.store.currentBranch().name;
|
|
4785
|
+
if (current !== failedBranch)
|
|
4786
|
+
return `active branch is ${current}`;
|
|
4787
|
+
try {
|
|
4788
|
+
await cm.switchBranch(sourceBranch);
|
|
4789
|
+
return `active branch restored to ${sourceBranch}`;
|
|
4790
|
+
}
|
|
4791
|
+
catch (restoreError) {
|
|
4792
|
+
const detail = restoreError instanceof Error ? restoreError.message : String(restoreError);
|
|
4793
|
+
const status = `RESTORE TO ${sourceBranch} ALSO FAILED (${detail}); agent is still on ${failedBranch}`;
|
|
4794
|
+
console.error(`[operator] agent=${agentName} ${status}`);
|
|
4795
|
+
return status;
|
|
4796
|
+
}
|
|
4797
|
+
}
|
|
4798
|
+
/**
|
|
4799
|
+
* Roll the active branch back so `messageId` becomes its tail: fork the
|
|
4800
|
+
* chronicle at that message (origin-sequence time-travel branch) and switch
|
|
4801
|
+
* to the fork. Everything after the message stays on the source branch.
|
|
4802
|
+
* Discord messages that left the live context get awareness markers via the
|
|
4803
|
+
* durable outbox, exactly as message-granular `undo` does.
|
|
4804
|
+
*
|
|
4805
|
+
* Throws `OperatorActionError` (`agent-busy`, `unknown-message`, …) — the
|
|
4806
|
+
* agent must be idle; nothing is queued.
|
|
4807
|
+
*/
|
|
4808
|
+
async rollbackToMessage(agentName, opts) {
|
|
4809
|
+
const logBase = {
|
|
4810
|
+
kind: 'rollback',
|
|
4811
|
+
agent: agentName,
|
|
4812
|
+
...(opts.requester ? { requester: opts.requester } : {}),
|
|
4813
|
+
...(opts.note ? { note: opts.note } : {}),
|
|
4814
|
+
params: { messageId: opts.messageId },
|
|
4815
|
+
};
|
|
4816
|
+
try {
|
|
4817
|
+
const agent = this.agents.get(agentName);
|
|
4818
|
+
if (!agent)
|
|
4819
|
+
throw new OperatorActionError('unknown-agent', `Unknown agent: ${agentName}`);
|
|
4820
|
+
const cm = agent.getContextManager();
|
|
4821
|
+
const total = cm.getMessageCount();
|
|
4822
|
+
const requestedIndex = locateMessageIndex(cm, opts.messageId);
|
|
4823
|
+
if (requestedIndex < 0) {
|
|
4824
|
+
throw new OperatorActionError('unknown-message', `Message ${opts.messageId} is not on the active branch`);
|
|
4825
|
+
}
|
|
4826
|
+
// Never bisect a body group: chronicle refuses that for removals
|
|
4827
|
+
// (byte-faithful reassembly needs the whole run) and branchAt has no
|
|
4828
|
+
// such guard of its own. Snap to the run's last shard.
|
|
4829
|
+
const run = bodyGroupRun(cm, requestedIndex);
|
|
4830
|
+
const targetIndex = run.to;
|
|
4831
|
+
const tailMessageId = run.toId;
|
|
4832
|
+
const messagesRemoved = total - targetIndex - 1;
|
|
4833
|
+
if (messagesRemoved <= 0) {
|
|
4834
|
+
throw new OperatorActionError('invalid', `Message ${opts.messageId} is already the tail of the active branch`);
|
|
4835
|
+
}
|
|
4836
|
+
const discarded = cm.getMessageWindow(targetIndex + 1, messagesRemoved, { resolveBlobs: false }).messages;
|
|
4837
|
+
const removedRefs = extractDiscordAwarenessRefs(discarded);
|
|
4838
|
+
const sourceBranch = this.store.currentBranch().name;
|
|
4839
|
+
const targetBranch = opts.branchName ?? `rollback/${agentName}/${Date.now()}`;
|
|
4840
|
+
if (targetBranch === sourceBranch) {
|
|
4841
|
+
throw new OperatorActionError('invalid', 'Rollback branch name must differ from the active branch');
|
|
4842
|
+
}
|
|
4843
|
+
// Gate + reserve the whole store (see reserveStoreForSurgery); held
|
|
4844
|
+
// until the switch has landed or been rolled back.
|
|
4845
|
+
const release = this.reserveStoreForSurgery('roll back', agentName);
|
|
4846
|
+
let branchName;
|
|
4847
|
+
try {
|
|
4848
|
+
// Prepare the external side effect before switching Chronicle. If the
|
|
4849
|
+
// process dies after the switch but before activate(), startup
|
|
4850
|
+
// promotes this batch by matching targetBranch to the active branch.
|
|
4851
|
+
// If the fork/switch itself throws, the batch is retired so it cannot
|
|
4852
|
+
// arm a later branch that happens to reuse the name.
|
|
4853
|
+
const markerBatch = this.discordAwarenessOutbox?.prepare({
|
|
4854
|
+
agentName,
|
|
4855
|
+
sourceBranch,
|
|
4856
|
+
targetBranch,
|
|
4857
|
+
refs: removedRefs,
|
|
4858
|
+
emoji: this.discordAwarenessEmoji,
|
|
4859
|
+
}) ?? null;
|
|
4860
|
+
try {
|
|
4861
|
+
branchName = cm.branchAt(tailMessageId, targetBranch);
|
|
4862
|
+
await cm.switchBranch(branchName);
|
|
4863
|
+
}
|
|
4864
|
+
catch (error) {
|
|
4865
|
+
// switchBranch moves the chronicle branch BEFORE awaiting strategy
|
|
4866
|
+
// initialization, so a rejection here can leave the store on the
|
|
4867
|
+
// new branch with an uninitialized strategy. Retire the batch and
|
|
4868
|
+
// go back to the source; keep the branch for diagnosis.
|
|
4869
|
+
if (markerBatch)
|
|
4870
|
+
this.discordAwarenessOutbox.discard(markerBatch.id);
|
|
4871
|
+
const restored = await this.restoreSourceBranch(cm, sourceBranch, targetBranch, agentName);
|
|
4872
|
+
throw new OperatorActionError('failed', `Rollback failed; ${restored}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
4873
|
+
}
|
|
4874
|
+
if (markerBatch)
|
|
4875
|
+
this.discordAwarenessOutbox.activate(markerBatch.id);
|
|
4876
|
+
await this.syncDiscordAwarenessMarkers();
|
|
4877
|
+
this.materializeConfigMountAfterBranchSwitch();
|
|
4878
|
+
}
|
|
4879
|
+
finally {
|
|
4880
|
+
release();
|
|
4881
|
+
}
|
|
4882
|
+
console.error(`[operator] rollback agent=${agentName} to=${tailMessageId}` +
|
|
4883
|
+
`${tailMessageId !== opts.messageId ? ` (requested ${opts.messageId}, snapped to end of body group)` : ''}` +
|
|
4884
|
+
` removed=${messagesRemoved} branch=${branchName} by=${describeRequester(opts.requester)}`);
|
|
4885
|
+
this.recordOperatorAction({
|
|
4886
|
+
...logBase,
|
|
4887
|
+
result: {
|
|
4888
|
+
sourceBranch,
|
|
4889
|
+
targetBranch: branchName,
|
|
4890
|
+
tailMessageId,
|
|
4891
|
+
messagesRemoved,
|
|
4892
|
+
discordRefs: removedRefs.length,
|
|
4893
|
+
},
|
|
4894
|
+
});
|
|
4895
|
+
return {
|
|
4896
|
+
agentName,
|
|
4897
|
+
sourceBranch,
|
|
4898
|
+
targetBranch: branchName,
|
|
4899
|
+
messagesRemoved,
|
|
4900
|
+
tailMessageId,
|
|
4901
|
+
removedRefs,
|
|
4902
|
+
lastVisible: await this.lastVisiblePreview(agentName),
|
|
4903
|
+
};
|
|
4904
|
+
}
|
|
4905
|
+
catch (error) {
|
|
4906
|
+
this.recordOperatorAction({ ...logBase, error: error instanceof Error ? error.message : String(error) });
|
|
4907
|
+
throw error;
|
|
4908
|
+
}
|
|
4909
|
+
}
|
|
4910
|
+
/**
|
|
4911
|
+
* Suppress specific messages from the live context without touching the
|
|
4912
|
+
* source branch: fork at the current head (`suppress/<agent>/<ts>`), redact
|
|
4913
|
+
* the messages on the fork, switch to it. Shards of a body group are always
|
|
4914
|
+
* removed together. Discord originals get awareness markers via the outbox;
|
|
4915
|
+
* the batch is `explicit`-activated only after every removal succeeded, and
|
|
4916
|
+
* a crash mid-way is finished at next boot (resumePreparedDiscordSuppressions).
|
|
4917
|
+
*
|
|
4918
|
+
* Not retroactive over derived state: a message already folded into an
|
|
4919
|
+
* autobiographical summary stays in that summary — roll back to before it
|
|
4920
|
+
* entered if that matters.
|
|
4921
|
+
*/
|
|
4922
|
+
async suppressMessages(agentName, opts) {
|
|
4923
|
+
const requestedIds = [...new Set(opts.messageIds.map(String))];
|
|
4924
|
+
const logBase = {
|
|
4925
|
+
kind: 'suppress',
|
|
4926
|
+
agent: agentName,
|
|
4927
|
+
...(opts.requester ? { requester: opts.requester } : {}),
|
|
4928
|
+
...(opts.note ? { note: opts.note } : {}),
|
|
4929
|
+
params: { messageIds: requestedIds },
|
|
4930
|
+
};
|
|
4931
|
+
try {
|
|
4932
|
+
const agent = this.agents.get(agentName);
|
|
4933
|
+
if (!agent)
|
|
4934
|
+
throw new OperatorActionError('unknown-agent', `Unknown agent: ${agentName}`);
|
|
4935
|
+
if (requestedIds.length === 0)
|
|
4936
|
+
throw new OperatorActionError('invalid', 'No message ids given');
|
|
4937
|
+
const cm = agent.getContextManager();
|
|
4938
|
+
const total = cm.getMessageCount();
|
|
4939
|
+
const located = locateMessageIndices(cm, new Set(requestedIds));
|
|
4940
|
+
const missing = requestedIds.filter((id) => !located.has(id));
|
|
4941
|
+
if (missing.length > 0) {
|
|
4942
|
+
throw new OperatorActionError('unknown-message', `Not on the active branch: ${missing.slice(0, 5).join(', ')}${missing.length > 5 ? ` (+${missing.length - 5})` : ''}`);
|
|
4943
|
+
}
|
|
4944
|
+
// Build removal intervals (index-ordered, newest first). A shard expands
|
|
4945
|
+
// to its whole body group — chronicle refuses to bisect one — and a
|
|
4946
|
+
// group interval is always removed as a RANGE: `removeMessage` refuses
|
|
4947
|
+
// any sharded message even when the group is down to one shard.
|
|
4948
|
+
const intervals = new Map();
|
|
4949
|
+
for (const id of requestedIds) {
|
|
4950
|
+
const run = bodyGroupRun(cm, located.get(id).index);
|
|
4951
|
+
intervals.set(`${run.from}-${run.to}`, run);
|
|
4952
|
+
}
|
|
4953
|
+
const ordered = [...intervals.values()].sort((a, b) => b.from - a.from);
|
|
4954
|
+
const messagesRemoved = ordered.reduce((n, iv) => n + (iv.to - iv.from + 1), 0);
|
|
4955
|
+
if (messagesRemoved >= total) {
|
|
4956
|
+
throw new OperatorActionError('invalid', `Cannot suppress all ${total} message(s) — at least one must remain`);
|
|
4957
|
+
}
|
|
4958
|
+
const targeted = ordered.flatMap((iv) => cm.getMessageWindow(iv.from, iv.to - iv.from + 1, { resolveBlobs: false }).messages);
|
|
4959
|
+
const removedIds = targeted.map((m) => String(m.id));
|
|
4960
|
+
const removedRefs = extractDiscordAwarenessRefs(targeted);
|
|
4961
|
+
const sourceBranch = this.store.currentBranch().name;
|
|
4962
|
+
const targetBranch = opts.branchName ?? `suppress/${agentName}/${Date.now()}`;
|
|
4963
|
+
if (targetBranch === sourceBranch) {
|
|
4964
|
+
throw new OperatorActionError('invalid', 'Suppression branch name must differ from the active branch');
|
|
4965
|
+
}
|
|
4966
|
+
// Gate + reserve the whole store (see reserveStoreForSurgery); held
|
|
4967
|
+
// until the fork is fully redacted or the source is restored.
|
|
4968
|
+
const release = this.reserveStoreForSurgery('suppress', agentName);
|
|
4969
|
+
let createdBranch;
|
|
4970
|
+
try {
|
|
4971
|
+
const markerBatch = this.discordAwarenessOutbox?.prepare({
|
|
4972
|
+
agentName,
|
|
4973
|
+
sourceBranch,
|
|
4974
|
+
targetBranch,
|
|
4975
|
+
refs: removedRefs,
|
|
4976
|
+
emoji: this.discordAwarenessEmoji,
|
|
4977
|
+
// Seeing targetBranch active does not prove the removals finished;
|
|
4978
|
+
// only this operation activates the batch, after the last redaction.
|
|
4979
|
+
activationPolicy: 'explicit',
|
|
4980
|
+
suppressionIntervals: ordered.map((iv) => ({ fromId: iv.fromId, toId: iv.toId })),
|
|
4981
|
+
}) ?? null;
|
|
4982
|
+
// Anything short of activation retires the batch: a prepared explicit
|
|
4983
|
+
// batch re-arms through preparedSuppressionsForBranch the moment its
|
|
4984
|
+
// target branch (or a descendant) becomes active — e.g. an operator
|
|
4985
|
+
// opening the failed fork to look — and an un-completable resume
|
|
4986
|
+
// there aborts framework start.
|
|
4987
|
+
const retireBatch = () => {
|
|
4988
|
+
if (markerBatch)
|
|
4989
|
+
this.discordAwarenessOutbox.discard(markerBatch.id);
|
|
4990
|
+
};
|
|
4991
|
+
try {
|
|
4992
|
+
createdBranch = await cm.fork(targetBranch);
|
|
4993
|
+
}
|
|
4994
|
+
catch (error) {
|
|
4995
|
+
// fork() switches the chronicle branch before awaiting strategy
|
|
4996
|
+
// initialization; a rejection can leave the store on the new
|
|
4997
|
+
// branch. Retire the batch and go back to the source.
|
|
4998
|
+
retireBatch();
|
|
4999
|
+
const restored = await this.restoreSourceBranch(cm, sourceBranch, targetBranch, agentName);
|
|
5000
|
+
throw new OperatorActionError('failed', `Suppression failed creating ${targetBranch}; ${restored}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
5001
|
+
}
|
|
5002
|
+
try {
|
|
5003
|
+
for (const iv of ordered) {
|
|
5004
|
+
if (iv.group || iv.fromId !== iv.toId)
|
|
5005
|
+
cm.removeMessages(iv.fromId, iv.toId);
|
|
5006
|
+
else
|
|
5007
|
+
cm.removeMessage(iv.fromId);
|
|
5008
|
+
}
|
|
5009
|
+
if (markerBatch)
|
|
5010
|
+
this.discordAwarenessOutbox.activate(markerBatch.id);
|
|
5011
|
+
}
|
|
5012
|
+
catch (error) {
|
|
5013
|
+
// A partially suppressed branch is not safe to serve from. Keep it
|
|
5014
|
+
// for diagnosis, put the agent back on the untouched source — and
|
|
5015
|
+
// say honestly whether that restore happened.
|
|
5016
|
+
retireBatch();
|
|
5017
|
+
const restored = await this.restoreSourceBranch(cm, sourceBranch, createdBranch, agentName);
|
|
5018
|
+
throw new OperatorActionError('failed', `Suppression failed on ${createdBranch}; ${restored}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
5019
|
+
}
|
|
5020
|
+
await this.syncDiscordAwarenessMarkers();
|
|
5021
|
+
this.materializeConfigMountAfterBranchSwitch();
|
|
5022
|
+
}
|
|
5023
|
+
finally {
|
|
5024
|
+
release();
|
|
5025
|
+
}
|
|
5026
|
+
console.error(`[operator] suppress agent=${agentName} removed=${messagesRemoved} branch=${createdBranch} ` +
|
|
5027
|
+
`by=${describeRequester(opts.requester)}`);
|
|
5028
|
+
const loggedIds = capIds(removedIds);
|
|
5029
|
+
this.recordOperatorAction({
|
|
5030
|
+
...logBase,
|
|
5031
|
+
result: {
|
|
5032
|
+
sourceBranch,
|
|
5033
|
+
targetBranch: createdBranch,
|
|
5034
|
+
messagesRemoved,
|
|
5035
|
+
removedIds: loggedIds.ids,
|
|
5036
|
+
...(loggedIds.truncated ? { removedIdsTruncated: true } : {}),
|
|
5037
|
+
discordRefs: removedRefs.length,
|
|
5038
|
+
},
|
|
5039
|
+
});
|
|
5040
|
+
return {
|
|
5041
|
+
agentName,
|
|
5042
|
+
sourceBranch,
|
|
5043
|
+
targetBranch: createdBranch,
|
|
5044
|
+
messagesRemoved,
|
|
5045
|
+
removedIds,
|
|
5046
|
+
removedRefs,
|
|
5047
|
+
lastVisible: await this.lastVisiblePreview(agentName),
|
|
5048
|
+
};
|
|
5049
|
+
}
|
|
5050
|
+
catch (error) {
|
|
5051
|
+
this.recordOperatorAction({ ...logBase, error: error instanceof Error ? error.message : String(error) });
|
|
5052
|
+
throw error;
|
|
5053
|
+
}
|
|
5054
|
+
}
|
|
5055
|
+
/** Re-materialize config files from the (new) active branch — fire and
|
|
5056
|
+
* forget; the gate picks changes up via mtime. Mirrors undoLastTurn. */
|
|
5057
|
+
materializeConfigMountAfterBranchSwitch() {
|
|
5058
|
+
const ws = this.moduleRegistry.getModule('workspace');
|
|
5059
|
+
if (ws && 'materializeMount' in ws) {
|
|
5060
|
+
ws
|
|
5061
|
+
.materializeMount('_config')
|
|
5062
|
+
.catch(() => { });
|
|
5063
|
+
}
|
|
5064
|
+
}
|
|
3456
5065
|
/**
|
|
3457
5066
|
* Check if undo/redo is available for an agent.
|
|
3458
5067
|
*/
|
|
@@ -3576,7 +5185,10 @@ export class AgentFramework {
|
|
|
3576
5185
|
while (!this.queue.isEmpty ||
|
|
3577
5186
|
// Direct inference requests (e.g. runEphemeralToCompletion) bypass the
|
|
3578
5187
|
// event queue — without this the loop can exit before they're drained.
|
|
3579
|
-
|
|
5188
|
+
// While quiesced, gate-parked requests are NOT progress and must not
|
|
5189
|
+
// block idle (only a budget restart can still run); identity-equivalent
|
|
5190
|
+
// to `.length > 0` when not quiesced.
|
|
5191
|
+
this.pendingRequests.some((r) => !this.quiesced || isTurnContinuation(r.reason)) ||
|
|
3580
5192
|
this.activeStreams.size > 0 ||
|
|
3581
5193
|
Array.from(this.agents.values()).some((a) => a.state.status !== 'idle')) {
|
|
3582
5194
|
await this.processNextEvent();
|
|
@@ -3805,7 +5417,7 @@ export class AgentFramework {
|
|
|
3805
5417
|
const deferred = this.drainDeferredFor(agent.name);
|
|
3806
5418
|
if (deferred.length > 0) {
|
|
3807
5419
|
for (const msg of deferred) {
|
|
3808
|
-
agent.getContextManager().addMessage(msg.participant, msg.content, msg.metadata);
|
|
5420
|
+
agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id));
|
|
3809
5421
|
// Injection guards: tool blocks would corrupt the tool-cycle
|
|
3810
5422
|
// structure the membrane enforces, and a message named as the
|
|
3811
5423
|
// agent itself would render as an ASSISTANT turn on the wire
|
|
@@ -3822,6 +5434,9 @@ export class AgentFramework {
|
|
|
3822
5434
|
}
|
|
3823
5435
|
}
|
|
3824
5436
|
}
|
|
5437
|
+
// Deferred writes handed to the store above: sync, then drop them
|
|
5438
|
+
// from the durable recovery queue (no-op unless one is persisted).
|
|
5439
|
+
this.ackDeferredWrites();
|
|
3825
5440
|
// A newly injected CONVERSATIONAL message begins a new
|
|
3826
5441
|
// conversational round inside the same provider inference. Remember
|
|
3827
5442
|
// that boundary for driveStream: an explicit send in the preceding
|
|
@@ -4680,7 +6295,7 @@ export class AgentFramework {
|
|
|
4680
6295
|
}
|
|
4681
6296
|
const providerCooldown = this.providerAccelerationCooldowns?.get(agentName);
|
|
4682
6297
|
if (providerCooldown) {
|
|
4683
|
-
if (now < providerCooldown.until) {
|
|
6298
|
+
if (now < providerCooldown.until || this.extendHostProviderHold(agentName, providerCooldown)) {
|
|
4684
6299
|
for (const req of requests) {
|
|
4685
6300
|
if (!providerCooldown.heldRequests.some((r) => this.sameInferenceRequest(r, req))) {
|
|
4686
6301
|
providerCooldown.heldRequests.push(req);
|
|
@@ -4717,16 +6332,60 @@ export class AgentFramework {
|
|
|
4717
6332
|
// This mirrors how the restart has always overwritten activeStreams
|
|
4718
6333
|
// rather than waiting for the old stream's teardown.
|
|
4719
6334
|
const budgetRestart = requests.find((r) => r.reason === 'context_budget_restart');
|
|
6335
|
+
// Host quiesce (issue #122): park wakes instead of starting turns —
|
|
6336
|
+
// the requeue mirrors the busy path below, so requests survive and
|
|
6337
|
+
// fire at resume. A context-budget restart passes through: it
|
|
6338
|
+
// CONTINUES a turn whose token is held (see the deadlock note below),
|
|
6339
|
+
// and quiesce's drain is exactly the phase where such a turn must be
|
|
6340
|
+
// allowed to finish. Parked requests are coalesced per reason (newest
|
|
6341
|
+
// kept): heartbeat/gate wakes arrive once per tick and nothing else
|
|
6342
|
+
// bounds a multi-day maintenance window's accumulation.
|
|
6343
|
+
if (this.quiesced) {
|
|
6344
|
+
// Continuations of a held turn (budget restart, non-streaming tool
|
|
6345
|
+
// results) pass; everything else parks.
|
|
6346
|
+
const passThrough = requests.filter((r) => isTurnContinuation(r.reason));
|
|
6347
|
+
const parked = requests.filter((r) => !isTurnContinuation(r.reason));
|
|
6348
|
+
// Coalesce per (reason, addressed): every channel wake shares one
|
|
6349
|
+
// reason, so keying on reason alone would let an ambient message
|
|
6350
|
+
// parked later evict a DM/mention parked earlier — and the
|
|
6351
|
+
// addressed-outranks-ambient trigger selection below would then
|
|
6352
|
+
// route the resumed turn into the wrong channel.
|
|
6353
|
+
const newestByKey = new Map();
|
|
6354
|
+
for (const request of parked) {
|
|
6355
|
+
const key = `${request.reason}|${request.addressed ? 'a' : 'n'}`;
|
|
6356
|
+
const prev = newestByKey.get(key);
|
|
6357
|
+
if (!prev || request.timestamp >= prev.timestamp) {
|
|
6358
|
+
newestByKey.set(key, request);
|
|
6359
|
+
}
|
|
6360
|
+
}
|
|
6361
|
+
this.pendingRequests.push(...newestByKey.values());
|
|
6362
|
+
if (passThrough.length === 0)
|
|
6363
|
+
continue;
|
|
6364
|
+
// The continuation continues its held turn; its parked siblings must
|
|
6365
|
+
// NOT be consumed by that turn — they were re-parked above, so narrow
|
|
6366
|
+
// the batch to the continuation(s) alone before the trigger selection.
|
|
6367
|
+
requests.length = 0;
|
|
6368
|
+
requests.push(...passThrough);
|
|
6369
|
+
}
|
|
4720
6370
|
const turnAlive = !budgetRestart && this.activeTurnTokens.has(agentName);
|
|
4721
6371
|
const providerGate = this.providerGates?.get(agentName);
|
|
4722
6372
|
// A primary can own provider admission while yielding to an already in-flight
|
|
4723
6373
|
// auxiliary call. Keep this agent's later wakes queued, but do not block the
|
|
4724
6374
|
// framework event loop or other residents while that auxiliary call settles.
|
|
4725
6375
|
const providerPrimaryWaiting = (providerGate?.primaryDepth ?? 0) > 0 && !this.activeTurnTokens.has(agentName);
|
|
4726
|
-
|
|
4727
|
-
|
|
6376
|
+
// A live surgery (rollback/suppress) holds the whole store: no turn may
|
|
6377
|
+
// start for ANY agent — including one registered after the reservation's
|
|
6378
|
+
// token snapshot — until the switch has landed or been rolled back.
|
|
6379
|
+
// Truthiness, not `!== null`: prototype-built harnesses (and any field
|
|
6380
|
+
// added later) leave the slot undefined, which must read as "not held".
|
|
6381
|
+
const surgeryHeld = !!this.surgeryHold;
|
|
6382
|
+
if (surgeryHeld || providerPrimaryWaiting || turnAlive || agent.state.status === 'inferring' || agent.state.status === 'streaming' || agent.state.status === 'waiting_for_tools') {
|
|
6383
|
+
// Re-queue requests, but warn if they've been pending too long.
|
|
6384
|
+
// Suppressed while quiesced: a drain-phase busy requeue is expected,
|
|
6385
|
+
// not a wedge tell.
|
|
4728
6386
|
const oldest = Math.min(...requests.map(r => r.timestamp));
|
|
4729
|
-
if (
|
|
6387
|
+
if (!this.quiesced &&
|
|
6388
|
+
now - oldest > STALE_REQUEST_MS &&
|
|
4730
6389
|
(this.staleWarnAt.get(agentName) ?? 0) < now - 60_000) {
|
|
4731
6390
|
// Trace and stderr share the throttle. The old code throttled only
|
|
4732
6391
|
// stderr, so a long-running tool call generated one trace per poll
|
|
@@ -4737,9 +6396,11 @@ export class AgentFramework {
|
|
|
4737
6396
|
// read 'idle' while a turn's teardown is pending, and a LEAKED turn
|
|
4738
6397
|
// token would look exactly like this — permanently requeued wakes.
|
|
4739
6398
|
// This line is the wedge's tell (idle+turn-alive, forever).
|
|
4740
|
-
const shownStatus =
|
|
4741
|
-
?
|
|
4742
|
-
: agent.state.status
|
|
6399
|
+
const shownStatus = surgeryHeld && agent.state.status === 'idle'
|
|
6400
|
+
? `idle+surgery(${this.surgeryHold.verb})`
|
|
6401
|
+
: turnAlive && agent.state.status === 'idle'
|
|
6402
|
+
? 'idle+turn-alive'
|
|
6403
|
+
: agent.state.status;
|
|
4743
6404
|
this.emitTrace({
|
|
4744
6405
|
type: 'inference:request_stale',
|
|
4745
6406
|
agentName,
|
|
@@ -5282,6 +6943,20 @@ export class AgentFramework {
|
|
|
5282
6943
|
console.error(`[provider-admission] ${agent.name}: turn-alive while parked — wake requeued, not started`);
|
|
5283
6944
|
return;
|
|
5284
6945
|
}
|
|
6946
|
+
// Same gap, second hazard (#122): the host may have QUIESCED while
|
|
6947
|
+
// this wake was parked. The scheduler's quiesce gate ran before the
|
|
6948
|
+
// park; starting here would bypass it and run a turn against a
|
|
6949
|
+
// context the operator believes frozen. Give admission back and
|
|
6950
|
+
// requeue — the scheduler re-parks it (coalesced) until resume.
|
|
6951
|
+
// Continuations of a held turn pass, as they do at the gate.
|
|
6952
|
+
if (this.quiesced && !isTurnContinuation(trigger?.reason ?? '')) {
|
|
6953
|
+
this.releasePrimaryProviderGate(agent.name);
|
|
6954
|
+
this.pendingRequests.push(trigger ?? {
|
|
6955
|
+
agentName: agent.name, reason: 'provider-admission:requeue', source: 'scheduler', timestamp: Date.now(),
|
|
6956
|
+
});
|
|
6957
|
+
console.error(`[provider-admission] ${agent.name}: host quiesced while parked — wake requeued, not started`);
|
|
6958
|
+
return;
|
|
6959
|
+
}
|
|
5285
6960
|
await this.startAgentStream(agent, trigger, attempt, true);
|
|
5286
6961
|
}).catch((error) => {
|
|
5287
6962
|
this.releasePrimaryProviderGate(agent.name);
|
|
@@ -5356,7 +7031,7 @@ export class AgentFramework {
|
|
|
5356
7031
|
// poison message — or a transient store-write failure — must not
|
|
5357
7032
|
// abort the turn or drop the messages behind it in the queue.
|
|
5358
7033
|
try {
|
|
5359
|
-
const id = agent.getContextManager().addMessage(msg.participant, msg.content, msg.metadata);
|
|
7034
|
+
const id = agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id));
|
|
5360
7035
|
this.emitTrace({ type: 'message:added', messageId: id, source: 'deferred-flush:turn-start' });
|
|
5361
7036
|
}
|
|
5362
7037
|
catch (err) {
|
|
@@ -5364,6 +7039,7 @@ export class AgentFramework {
|
|
|
5364
7039
|
`(participant=${msg.participant}):`, err);
|
|
5365
7040
|
}
|
|
5366
7041
|
}
|
|
7042
|
+
this.ackDeferredWrites();
|
|
5367
7043
|
}
|
|
5368
7044
|
}
|
|
5369
7045
|
// Record turn checkpoint before inference (only on first attempt, not retries)
|
|
@@ -6419,6 +8095,40 @@ export class AgentFramework {
|
|
|
6419
8095
|
break;
|
|
6420
8096
|
this.abortAgentScript(agent.name, 'stream error');
|
|
6421
8097
|
agent.reset();
|
|
8098
|
+
// A quiesce-abandoned cancel may surface as `error` instead of
|
|
8099
|
+
// `aborted` depending on how the stream implementation reports
|
|
8100
|
+
// the cancellation. Same contract as the aborted branch: settle
|
|
8101
|
+
// honestly, no errorPolicy retry (which would relaunch inference
|
|
8102
|
+
// mid-maintenance-window), no inference:exhausted (which feeds
|
|
8103
|
+
// the failure streak / hard-down / poison-history accounting).
|
|
8104
|
+
{
|
|
8105
|
+
const cancelKey = `${agent.name}:${myStreamId}`;
|
|
8106
|
+
if (this.frameworkCancelledStreams.get(cancelKey) === 'quiesce_abandoned') {
|
|
8107
|
+
this.frameworkCancelledStreams.delete(cancelKey);
|
|
8108
|
+
// Same terminal as the `aborted` twin: §10.5 lifecycle reads
|
|
8109
|
+
// 'aborted' (not 'completed'), and the gate is released only
|
|
8110
|
+
// for THIS agent instance — a name re-registered meanwhile
|
|
8111
|
+
// (ephemeral disposal, conversation replacement) owns its own
|
|
8112
|
+
// gate liveness. The inference-log terminal was already
|
|
8113
|
+
// written at the top of this case (`Stream error`).
|
|
8114
|
+
lifecyclePhase = 'aborted';
|
|
8115
|
+
this.settleAgent(agent.name, {
|
|
8116
|
+
stopReason: 'exhausted',
|
|
8117
|
+
speech: '',
|
|
8118
|
+
error: 'Turn abandoned by operator quiesce',
|
|
8119
|
+
});
|
|
8120
|
+
this.emitTrace({
|
|
8121
|
+
type: 'inference:aborted',
|
|
8122
|
+
agentName: agent.name,
|
|
8123
|
+
durationMs,
|
|
8124
|
+
reason: 'quiesce_abandoned',
|
|
8125
|
+
});
|
|
8126
|
+
if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) {
|
|
8127
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
8128
|
+
}
|
|
8129
|
+
break;
|
|
8130
|
+
}
|
|
8131
|
+
}
|
|
6422
8132
|
if (ownsProviderGate && this.holdProviderAcceleration(agent, err, trigger)) {
|
|
6423
8133
|
lifecyclePhase = 'failed';
|
|
6424
8134
|
if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId)
|
|
@@ -6476,6 +8186,30 @@ export class AgentFramework {
|
|
|
6476
8186
|
else if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) {
|
|
6477
8187
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
6478
8188
|
}
|
|
8189
|
+
if (cancelKind === 'quiesce_abandoned') {
|
|
8190
|
+
// Operator-initiated cancel during a quiesce drain. Settle
|
|
8191
|
+
// the turn honestly — deliberately NO inference:exhausted:
|
|
8192
|
+
// that trace centrally drives the consecutive-failure
|
|
8193
|
+
// streak, hard-down ops alerts, and the poison-history
|
|
8194
|
+
// breaker, none of which an operator cancel represents.
|
|
8195
|
+
// inference:aborted carries the observability instead.
|
|
8196
|
+
if (agent.streamId === myStreamId) {
|
|
8197
|
+
this.abortAgentScript(agent.name, 'turn abandoned by quiesce');
|
|
8198
|
+
agent.reset();
|
|
8199
|
+
this.settleAgent(agent.name, {
|
|
8200
|
+
stopReason: 'exhausted',
|
|
8201
|
+
speech: '',
|
|
8202
|
+
error: 'Turn abandoned by operator quiesce',
|
|
8203
|
+
});
|
|
8204
|
+
}
|
|
8205
|
+
this.emitTrace({
|
|
8206
|
+
type: 'inference:aborted',
|
|
8207
|
+
agentName: agent.name,
|
|
8208
|
+
durationMs: Date.now() - startTime,
|
|
8209
|
+
reason: 'quiesce_abandoned',
|
|
8210
|
+
});
|
|
8211
|
+
return;
|
|
8212
|
+
}
|
|
6479
8213
|
// endTurn IS a logical turn end — earlier rounds may have
|
|
6480
8214
|
// live-routed prose (narrate → skip_reply is a real shape),
|
|
6481
8215
|
// so settle the delivery chain and drop the receipt. A
|
|
@@ -6749,8 +8483,12 @@ export class AgentFramework {
|
|
|
6749
8483
|
if (frameReachedTerminal && this.deferredMessages.length > 0 && this.pendingAssistantBlocks.size === 0) {
|
|
6750
8484
|
const deferred = this.drainDeferredFor(agent.name);
|
|
6751
8485
|
for (const msg of deferred) {
|
|
6752
|
-
this.addMessage(msg.participant, msg.content, msg.metadata,
|
|
8486
|
+
this.addMessage(msg.participant, msg.content, msg.metadata, {
|
|
8487
|
+
deferredWriteId: msg.id,
|
|
8488
|
+
...(msg.forAgent ? { forAgent: msg.forAgent } : {}),
|
|
8489
|
+
});
|
|
6753
8490
|
}
|
|
8491
|
+
this.ackDeferredWrites();
|
|
6754
8492
|
}
|
|
6755
8493
|
}
|
|
6756
8494
|
}
|
|
@@ -6816,6 +8554,9 @@ export class AgentFramework {
|
|
|
6816
8554
|
// Idle AND no turn alive: status reads 'idle' from dequeue until the
|
|
6817
8555
|
// stream registers, and again while a turn's teardown is pending
|
|
6818
8556
|
// (the scheduler's own busy test, 'idle+turn-alive').
|
|
8557
|
+
if (this.surgeryHold) {
|
|
8558
|
+
throw new Error(`puppet refused: the store is under live ${this.surgeryHold.verb} for ${this.surgeryHold.agentName} — retry when it completes`);
|
|
8559
|
+
}
|
|
6819
8560
|
if (agent.state.status !== 'idle' || this.activeTurnTokens.has(agentName)) {
|
|
6820
8561
|
const shown = agent.state.status === 'idle' ? 'idle+turn-alive' : agent.state.status;
|
|
6821
8562
|
throw new Error(`puppet refused: agent ${agentName} is ${shown} (requires idle — ` +
|
|
@@ -6881,7 +8622,7 @@ export class AgentFramework {
|
|
|
6881
8622
|
cm.addMessage('user', blocks);
|
|
6882
8623
|
}
|
|
6883
8624
|
else {
|
|
6884
|
-
this.deferredMessages.push({ participant: agentName, content: toolUse, forAgent: agentName }, { participant: 'user', content: blocks, forAgent: agentName });
|
|
8625
|
+
this.deferredMessages.push({ id: randomUUID(), seq: ++this.deferredSeq, participant: agentName, content: toolUse, forAgent: agentName }, { id: randomUUID(), seq: ++this.deferredSeq, participant: 'user', content: blocks, forAgent: agentName });
|
|
6885
8626
|
}
|
|
6886
8627
|
this.emitTrace({
|
|
6887
8628
|
type: 'puppet:tool-call',
|
|
@@ -6909,8 +8650,12 @@ export class AgentFramework {
|
|
|
6909
8650
|
if (!this.activeTurnTokens.has(agentName)
|
|
6910
8651
|
&& this.deferredMessages.length > 0 && this.pendingAssistantBlocks.size === 0) {
|
|
6911
8652
|
for (const msg of this.drainDeferredFor(agentName)) {
|
|
6912
|
-
this.addMessage(msg.participant, msg.content, msg.metadata,
|
|
8653
|
+
this.addMessage(msg.participant, msg.content, msg.metadata, {
|
|
8654
|
+
deferredWriteId: msg.id,
|
|
8655
|
+
...(msg.forAgent ? { forAgent: msg.forAgent } : {}),
|
|
8656
|
+
});
|
|
6913
8657
|
}
|
|
8658
|
+
this.ackDeferredWrites();
|
|
6914
8659
|
}
|
|
6915
8660
|
}
|
|
6916
8661
|
}
|
|
@@ -8010,6 +9755,10 @@ export class AgentFramework {
|
|
|
8010
9755
|
// their own wake), at the agent's next tool boundary (where they are
|
|
8011
9756
|
// ALSO injected into the live stream — hear-while-acting), or in
|
|
8012
9757
|
// driveStream's finally when the turn ends.
|
|
9758
|
+
// Quiesce (issue #122): the window exists to run compression/refold
|
|
9759
|
+
// against a FROZEN context — module events and api message.send must not
|
|
9760
|
+
// append mid-surgery. Deferred here, flushed by resume(). tool_result
|
|
9761
|
+
// still lands (a draining turn's continuation depends on it).
|
|
8013
9762
|
const hasToolResult = content.some(b => b.type === 'tool_result');
|
|
8014
9763
|
// Explicitly-targeted deliveries scope the tool-cycle check to the
|
|
8015
9764
|
// target (pendingAssistantBlocks is keyed by agent); the default path
|
|
@@ -8019,13 +9768,25 @@ export class AgentFramework {
|
|
|
8019
9768
|
? this.pendingAssistantBlocks.has(agent.name)
|
|
8020
9769
|
: this.pendingAssistantBlocks.size > 0;
|
|
8021
9770
|
if (!hasToolResult &&
|
|
8022
|
-
(
|
|
9771
|
+
(this.quiesced ||
|
|
9772
|
+
midToolCycle ||
|
|
8023
9773
|
this.activeTurnTokens.has(agent.name) ||
|
|
8024
9774
|
this.activeStreams.has(agent.name))) {
|
|
8025
|
-
|
|
9775
|
+
// A re-deferral of a drained entry keeps its id and leaves the un-acked
|
|
9776
|
+
// set: one logical message, one durable identity — never two entries.
|
|
9777
|
+
const id = opts?.deferredWriteId ?? randomUUID();
|
|
9778
|
+
let seq;
|
|
9779
|
+
if (opts?.deferredWriteId) {
|
|
9780
|
+
const prior = this.unackedDeferredWrites.find((m) => m.id === id);
|
|
9781
|
+
seq = prior?.seq;
|
|
9782
|
+
this.unackedDeferredWrites = this.unackedDeferredWrites.filter((m) => m.id !== id);
|
|
9783
|
+
}
|
|
9784
|
+
this.deferredMessages.push({ id, seq: seq ?? ++this.deferredSeq, participant, content, metadata, forAgent: opts?.forAgent });
|
|
9785
|
+
if (this.quiesced || this.deferredWritesPersisted)
|
|
9786
|
+
this.persistDeferredWrites();
|
|
8026
9787
|
return ''; // Deferred — flushed at the target's next boundary
|
|
8027
9788
|
}
|
|
8028
|
-
return agent.getContextManager().addMessage(participant, content, metadata);
|
|
9789
|
+
return agent.getContextManager().addMessage(participant, content, opts?.deferredWriteId ? withDeferredWriteId(metadata, opts.deferredWriteId) : metadata);
|
|
8029
9790
|
}
|
|
8030
9791
|
/**
|
|
8031
9792
|
* Splice out the deferred messages addressed to `agentName` (explicitly
|
|
@@ -8042,6 +9803,12 @@ export class AgentFramework {
|
|
|
8042
9803
|
(target === agentName ? mine : rest).push(msg);
|
|
8043
9804
|
}
|
|
8044
9805
|
this.deferredMessages = rest;
|
|
9806
|
+
mine.sort(bySeq);
|
|
9807
|
+
// The drained messages are about to be written by the caller; they stay
|
|
9808
|
+
// in the DURABLE queue (as un-acked) until the caller's
|
|
9809
|
+
// ackDeferredWrites() has synced the chronicle. Removing them here would
|
|
9810
|
+
// acknowledge before the write is durable.
|
|
9811
|
+
this.handOffDeferredWrites(mine);
|
|
8045
9812
|
return mine;
|
|
8046
9813
|
}
|
|
8047
9814
|
editMessage(id, content) {
|
|
@@ -8179,6 +9946,8 @@ export class AgentFramework {
|
|
|
8179
9946
|
auxiliaryInFlight: gate?.auxiliaryInFlight ?? 0,
|
|
8180
9947
|
auxiliaryDeferred: gate?.deferredAuxiliary ?? 0,
|
|
8181
9948
|
cooldownUntil: cooldown?.until ?? null,
|
|
9949
|
+
cooldownReason: cooldown?.reason ?? null,
|
|
9950
|
+
hostHold: cooldown?.hostHoldError !== undefined,
|
|
8182
9951
|
heldRequests: cooldown?.heldRequests.length ?? 0,
|
|
8183
9952
|
lastRecovery: this.providerAccelerationLastRecovery?.get(name) ?? null,
|
|
8184
9953
|
};
|
|
@@ -8850,9 +10619,22 @@ export class AgentFramework {
|
|
|
8850
10619
|
for (const connection of this.mcplServerRegistry?.getAllServers() ?? []) {
|
|
8851
10620
|
// ready() can synchronously flush a nested list-change notification that
|
|
8852
10621
|
// installs a newer global generation. Never let this older completion
|
|
8853
|
-
// release any remaining server behind that newer gate.
|
|
10622
|
+
// release any remaining server behind that newer gate. (The quiesce
|
|
10623
|
+
// check sits AFTER this guard: readyControlPlane can also synchronously
|
|
10624
|
+
// flush a control event that installs a newer barrier.)
|
|
8854
10625
|
if (this.discordAwarenessBarrier !== null)
|
|
8855
10626
|
return false;
|
|
10627
|
+
// Host quiesce (issue #122): this is the single point every barrier
|
|
10628
|
+
// completion and reconnect flow funnels through, so holding here keeps
|
|
10629
|
+
// data planes paused across awareness drains and reconnects for the
|
|
10630
|
+
// whole window. Control planes come up normally (host/command rides the
|
|
10631
|
+
// control plane, so resume stays deliverable); resume() performs the
|
|
10632
|
+
// real ready() flush through this same funnel once its feasibility gate
|
|
10633
|
+
// passes and the flag is cleared.
|
|
10634
|
+
if (this.quiesced) {
|
|
10635
|
+
connection.readyControlPlane();
|
|
10636
|
+
continue;
|
|
10637
|
+
}
|
|
8856
10638
|
connection.ready();
|
|
8857
10639
|
}
|
|
8858
10640
|
return this.discordAwarenessBarrier === null;
|