@animalabs/agent-framework 0.22.0 → 0.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -4
- package/dist/src/agent.d.ts.map +1 -1
- package/dist/src/agent.js +3 -1
- package/dist/src/agent.js.map +1 -1
- package/dist/src/api/server.d.ts.map +1 -1
- package/dist/src/api/server.js +41 -4
- package/dist/src/api/server.js.map +1 -1
- package/dist/src/framework.d.ts +319 -20
- package/dist/src/framework.d.ts.map +1 -1
- package/dist/src/framework.js +1500 -182
- package/dist/src/framework.js.map +1 -1
- package/dist/src/index.d.ts +2 -2
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/mcpl/index.d.ts +2 -0
- package/dist/src/mcpl/index.d.ts.map +1 -1
- package/dist/src/mcpl/index.js.map +1 -1
- package/dist/src/mcpl/reference-fetcher.d.ts +2 -1
- package/dist/src/mcpl/reference-fetcher.d.ts.map +1 -1
- package/dist/src/mcpl/reference-fetcher.js +20 -9
- package/dist/src/mcpl/reference-fetcher.js.map +1 -1
- package/dist/src/mcpl/references.d.ts +29 -6
- package/dist/src/mcpl/references.d.ts.map +1 -1
- package/dist/src/mcpl/references.js +57 -16
- package/dist/src/mcpl/references.js.map +1 -1
- package/dist/src/mcpl/server-connection.d.ts +28 -1
- package/dist/src/mcpl/server-connection.d.ts.map +1 -1
- package/dist/src/mcpl/server-connection.js +32 -4
- package/dist/src/mcpl/server-connection.js.map +1 -1
- package/dist/src/module-registry.d.ts +9 -3
- package/dist/src/module-registry.d.ts.map +1 -1
- package/dist/src/module-registry.js +3 -2
- package/dist/src/module-registry.js.map +1 -1
- package/dist/src/modules/workspace/index.d.ts.map +1 -1
- package/dist/src/modules/workspace/index.js +46 -4
- package/dist/src/modules/workspace/index.js.map +1 -1
- package/dist/src/modules/workspace/observe.d.ts +13 -3
- package/dist/src/modules/workspace/observe.d.ts.map +1 -1
- package/dist/src/modules/workspace/observe.js +24 -4
- package/dist/src/modules/workspace/observe.js.map +1 -1
- package/dist/src/modules/workspace/reconcile.d.ts +14 -0
- package/dist/src/modules/workspace/reconcile.d.ts.map +1 -1
- package/dist/src/modules/workspace/reconcile.js +10 -2
- package/dist/src/modules/workspace/reconcile.js.map +1 -1
- package/dist/src/operator-log.d.ts +41 -0
- package/dist/src/operator-log.d.ts.map +1 -1
- package/dist/src/operator-log.js.map +1 -1
- package/dist/src/provider-waits.d.ts +216 -0
- package/dist/src/provider-waits.d.ts.map +1 -0
- package/dist/src/provider-waits.js +482 -0
- package/dist/src/provider-waits.js.map +1 -0
- package/dist/src/refusal-reactions.d.ts +14 -12
- package/dist/src/refusal-reactions.d.ts.map +1 -1
- package/dist/src/refusal-reactions.js +14 -12
- package/dist/src/refusal-reactions.js.map +1 -1
- package/dist/src/tool-result-guard.d.ts +30 -13
- package/dist/src/tool-result-guard.d.ts.map +1 -1
- package/dist/src/tool-result-guard.js +71 -25
- package/dist/src/tool-result-guard.js.map +1 -1
- package/dist/src/types/events.d.ts +12 -0
- package/dist/src/types/events.d.ts.map +1 -1
- package/dist/src/types/module.d.ts +12 -4
- package/dist/src/types/module.d.ts.map +1 -1
- package/dist/src/types/trace.d.ts +14 -3
- package/dist/src/types/trace.d.ts.map +1 -1
- package/dist/test/discord-awareness-outbox.test.js +3 -0
- package/dist/test/discord-awareness-outbox.test.js.map +1 -1
- package/dist/test/explicit-prose-routing.test.js +0 -2
- package/dist/test/explicit-prose-routing.test.js.map +1 -1
- package/dist/test/framework.test.js +160 -1
- package/dist/test/framework.test.js.map +1 -1
- package/dist/test/helpers/coalescing-fixture.d.ts.map +1 -1
- package/dist/test/helpers/coalescing-fixture.js +3 -2
- package/dist/test/helpers/coalescing-fixture.js.map +1 -1
- package/dist/test/inference-exhaustion-rewind.test.js +142 -14
- package/dist/test/inference-exhaustion-rewind.test.js.map +1 -1
- package/dist/test/inference-failure-bounds.test.d.ts +2 -0
- package/dist/test/inference-failure-bounds.test.d.ts.map +1 -0
- package/dist/test/inference-failure-bounds.test.js +1201 -0
- package/dist/test/inference-failure-bounds.test.js.map +1 -0
- package/dist/test/late-events-after-stop.test.d.ts +2 -0
- package/dist/test/late-events-after-stop.test.d.ts.map +1 -0
- package/dist/test/late-events-after-stop.test.js +122 -0
- package/dist/test/late-events-after-stop.test.js.map +1 -0
- package/dist/test/mcpl-awareness-barrier.test.js +141 -0
- package/dist/test/mcpl-awareness-barrier.test.js.map +1 -1
- package/dist/test/mcpl-coalescing.test.js +78 -16
- package/dist/test/mcpl-coalescing.test.js.map +1 -1
- package/dist/test/mcpl-config-diagnostics.test.js +6 -3
- package/dist/test/mcpl-config-diagnostics.test.js.map +1 -1
- package/dist/test/mcpl-manifest-changed.test.js +2 -1
- package/dist/test/mcpl-manifest-changed.test.js.map +1 -1
- package/dist/test/mcpl-reference-fetcher.test.js +232 -4
- package/dist/test/mcpl-reference-fetcher.test.js.map +1 -1
- package/dist/test/mcpl-reference-fields.test.js +69 -2
- package/dist/test/mcpl-reference-fields.test.js.map +1 -1
- package/dist/test/mcpl-ws-transport.test.js +103 -0
- package/dist/test/mcpl-ws-transport.test.js.map +1 -1
- package/dist/test/ops-alert.test.js +2 -1
- package/dist/test/ops-alert.test.js.map +1 -1
- package/dist/test/placed-reaction-baseline.test.d.ts +2 -0
- package/dist/test/placed-reaction-baseline.test.d.ts.map +1 -0
- package/dist/test/placed-reaction-baseline.test.js +312 -0
- package/dist/test/placed-reaction-baseline.test.js.map +1 -0
- package/dist/test/provider-acceleration-cooldown.test.js +5 -1
- package/dist/test/provider-acceleration-cooldown.test.js.map +1 -1
- package/dist/test/provider-waits.test.d.ts +2 -0
- package/dist/test/provider-waits.test.d.ts.map +1 -0
- package/dist/test/provider-waits.test.js +2196 -0
- package/dist/test/provider-waits.test.js.map +1 -0
- package/dist/test/reaction-baseline-platform.test.d.ts +2 -0
- package/dist/test/reaction-baseline-platform.test.d.ts.map +1 -0
- package/dist/test/reaction-baseline-platform.test.js +124 -0
- package/dist/test/reaction-baseline-platform.test.js.map +1 -0
- package/dist/test/refusal-reaction-baseline.test.js +10 -3
- package/dist/test/refusal-reaction-baseline.test.js.map +1 -1
- package/dist/test/save-recent-image.test.js +138 -2
- package/dist/test/save-recent-image.test.js.map +1 -1
- package/dist/test/surgery-marks-receipt.test.d.ts +2 -0
- package/dist/test/surgery-marks-receipt.test.d.ts.map +1 -0
- package/dist/test/surgery-marks-receipt.test.js +540 -0
- package/dist/test/surgery-marks-receipt.test.js.map +1 -0
- package/dist/test/tool-result-guard.test.js +221 -7
- package/dist/test/tool-result-guard.test.js.map +1 -1
- package/dist/test/workspace-three-way.test.js +115 -0
- package/dist/test/workspace-three-way.test.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/agent.ts +4 -1
- package/src/api/server.ts +43 -4
- package/src/framework.ts +1538 -189
- package/src/index.ts +7 -2
- package/src/mcpl/index.ts +2 -0
- package/src/mcpl/reference-fetcher.ts +18 -8
- package/src/mcpl/references.ts +57 -13
- package/src/mcpl/server-connection.ts +55 -4
- package/src/module-registry.ts +17 -9
- package/src/modules/workspace/index.ts +44 -4
- package/src/modules/workspace/observe.ts +31 -5
- package/src/modules/workspace/reconcile.ts +23 -2
- package/src/operator-log.ts +30 -0
- package/src/provider-waits.ts +523 -0
- package/src/refusal-reactions.ts +14 -12
- package/src/tool-result-guard.ts +72 -24
- package/src/types/events.ts +12 -0
- package/src/types/module.ts +12 -4
- package/src/types/trace.ts +14 -3
package/dist/src/framework.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ToolPresentation, presentationTools, isPresentationTool, renderCatalogue } from "./tool-presentation.js";
|
|
2
2
|
import { dirname, join } from 'node:path';
|
|
3
|
-
import { INLINE_WITHHELD_TEXT, classifyBlock, isInlineContradiction, referenceRegistry, referenceStubOrNull } from './mcpl/references.js';
|
|
3
|
+
import { EXPIRY_REASON, INLINE_WITHHELD_TEXT, REFERENCE_LIMITS, classifyBlock, isInlineContradiction, isReferenceExpired, referenceExpiry, referenceRegistry, referenceStubOrNull, referenceTypeLabel, sanitizeLabel } from './mcpl/references.js';
|
|
4
4
|
import { ReferenceFetcher, DEFAULT_FETCH_MAX_BYTES, EAGER_FETCH_TIMEOUT_MS } from './mcpl/reference-fetcher.js';
|
|
5
5
|
import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
|
6
6
|
import { JsStore } from '@animalabs/chronicle';
|
|
@@ -10,7 +10,7 @@ import { SUBCONSCIOUS_TOOLS, SUBCONSCIOUS_TOOL_NAMES } from './tune-out/tools.js
|
|
|
10
10
|
import { TuneOutCoordinator, TUNE_OUT_DEFAULTS } from './tune-out/coordinator.js';
|
|
11
11
|
import { ProcessQueueImpl } from './queue.js';
|
|
12
12
|
import { DEFAULT_SHUTDOWN_TIMEOUT_MS, validateShutdownTimeout, shutdownErrorMessage } from './shutdown.js';
|
|
13
|
-
import { REFUSAL_REACTIONS, REFUSAL_REACTION_FALLBACK } from './refusal-reactions.js';
|
|
13
|
+
import { REFUSAL_REACTIONS, REFUSAL_REACTION_FALLBACK, REFUSAL_REACTION_BASELINE } from './refusal-reactions.js';
|
|
14
14
|
import { Agent } from './agent.js';
|
|
15
15
|
import { ModuleRegistry, checkedToolResult, isStateExistsError, storableToolResult, thrownResult } from './module-registry.js';
|
|
16
16
|
import { McplServerRegistry } from './mcpl/server-registry.js';
|
|
@@ -31,6 +31,7 @@ import { InferenceRouter } from './mcpl/inference-router.js';
|
|
|
31
31
|
import { ChannelRegistry } from './mcpl/channel-registry.js';
|
|
32
32
|
import { ConversationRouter, CONVERSATION_ROUTING_DEPRECATION_NOTICE } from './mcpl/conversation-router.js';
|
|
33
33
|
import { safeSlice } from './safe-slice.js';
|
|
34
|
+
import { ProviderWaits, EVERY_MODEL, isReleaseModel, withoutControls } from './provider-waits.js';
|
|
34
35
|
import { nonErrorResultText, toolResultDataToHistoryString, truncateForHistory, DEFAULT_TOOL_RESULT_INLINE_MAX_CHARS, } from './tool-result-history.js';
|
|
35
36
|
import { TOOL_IMAGE_REF_RE, ToolImageLedger, formatPreservedImageSlots, parseImagePlaceholders, sha256Hex, splitPreservingImageSlots, } from './tool-image-ledger.js';
|
|
36
37
|
import { randomUUID, createHash } from 'node:crypto';
|
|
@@ -290,6 +291,7 @@ import { CheckpointManager } from './mcpl/checkpoint-manager.js';
|
|
|
290
291
|
import { isToolAllowed } from './mcpl/tool-policy.js';
|
|
291
292
|
import { EventGate, formatShadowWarning } from './gate/event-gate.js';
|
|
292
293
|
import { UsageTracker } from './usage/usage-tracker.js';
|
|
294
|
+
import { isWebSocketTransport } from './mcpl/transport.js';
|
|
293
295
|
import { formatZonedDateTime, resolveTimeZone } from './timezone.js';
|
|
294
296
|
import { DEFAULT_DISCORD_AWARENESS_EMOJI, DiscordAwarenessOutbox, defaultDiscordAwarenessOutboxPath, extractDiscordAwarenessRefs, } from './recovery/discord-awareness-outbox.js';
|
|
295
297
|
import { OperatorActionError, OperatorLog, capIds, defaultOperatorLogPath, } from './operator-log.js';
|
|
@@ -361,6 +363,20 @@ const MAX_TURN_CHECKPOINTS = 20;
|
|
|
361
363
|
const DEFAULT_DISCORD_AWARENESS_DEADLINE_MS = 10_000;
|
|
362
364
|
const MIN_DISCORD_AWARENESS_DEADLINE_MS = 50;
|
|
363
365
|
const MAX_DISCORD_AWARENESS_DEADLINE_MS = 60_000;
|
|
366
|
+
/** Several drains' outcomes as one: unavailable only if every one was. */
|
|
367
|
+
function combineDrainOutcomes(outcomes) {
|
|
368
|
+
if (outcomes.every((outcome) => outcome.status === 'unavailable')) {
|
|
369
|
+
return {
|
|
370
|
+
status: 'unavailable',
|
|
371
|
+
accounted: outcomes.reduce((count, outcome) => count + (outcome.status === 'unavailable' ? outcome.accounted : 0), 0),
|
|
372
|
+
};
|
|
373
|
+
}
|
|
374
|
+
return {
|
|
375
|
+
status: 'delivered',
|
|
376
|
+
delivered: outcomes.reduce((count, outcome) => count + (outcome.status === 'delivered' ? outcome.delivered : 0), 0),
|
|
377
|
+
failed: outcomes.reduce((count, outcome) => count + (outcome.status === 'delivered' ? outcome.failed : 0), 0),
|
|
378
|
+
};
|
|
379
|
+
}
|
|
364
380
|
function normalizeDiscordAwarenessDeadline(value) {
|
|
365
381
|
if (value === undefined || !Number.isFinite(value) || value <= 0) {
|
|
366
382
|
return DEFAULT_DISCORD_AWARENESS_DEADLINE_MS;
|
|
@@ -459,11 +475,35 @@ function bodyGroupRun(cm, index) {
|
|
|
459
475
|
group: win.messages[0].bodyGroupId !== undefined,
|
|
460
476
|
};
|
|
461
477
|
}
|
|
478
|
+
function describeMarkers(m) {
|
|
479
|
+
if (m.status === 'queued')
|
|
480
|
+
return `queued:${m.queued}`;
|
|
481
|
+
if (m.status === 'not-scheduled')
|
|
482
|
+
return 'not-scheduled(batch retired)';
|
|
483
|
+
if (m.status === 'unresolved')
|
|
484
|
+
return `unresolved(batch ${m.batchId} may still be delivered)`;
|
|
485
|
+
return 'none';
|
|
486
|
+
}
|
|
462
487
|
function describeRequester(r) {
|
|
463
488
|
if (!r)
|
|
464
489
|
return 'unknown';
|
|
465
490
|
return `${r.name ?? r.id ?? 'unknown'} (${r.via})`;
|
|
466
491
|
}
|
|
492
|
+
/** What a held call or turn says of the wait that holds it: no call was
|
|
493
|
+
* made. The refusal of an auxiliary call and the trace of a turn held once
|
|
494
|
+
* compiled say it the same way. */
|
|
495
|
+
function providerWaitText(agentName, model, wait) {
|
|
496
|
+
return `Provider wait: ${model} is held for ${agentName} ` +
|
|
497
|
+
`${wait.until === null ? 'until an operator releases it' : `until ${new Date(wait.until).toISOString()}`} ` +
|
|
498
|
+
`(recorded ${new Date(wait.setAt).toISOString()}: ${wait.reason}); no call was made`;
|
|
499
|
+
}
|
|
500
|
+
/** Who asked for an operator act, as a durable record and a log line carry
|
|
501
|
+
* it. The name comes from a caller, so a control character (a forged log
|
|
502
|
+
* line) becomes a space, and it is bounded. */
|
|
503
|
+
function operatorLabel(by) {
|
|
504
|
+
const label = safeSlice(withoutControls(by).trim(), 0, 120);
|
|
505
|
+
return label === '' ? 'operator' : label;
|
|
506
|
+
}
|
|
467
507
|
function hostCommandRequester(serverId, params) {
|
|
468
508
|
return {
|
|
469
509
|
via: `host-command:${serverId}`,
|
|
@@ -490,11 +530,11 @@ function isPermanentDiscordReactionFailure(message) {
|
|
|
490
530
|
class DefaultErrorPolicy {
|
|
491
531
|
maxRetries = 3;
|
|
492
532
|
onInferenceError(error, _agentName, attempt) {
|
|
493
|
-
if (error
|
|
533
|
+
if (isMembraneError(error) && error.retryable === false) {
|
|
494
534
|
return { retry: false };
|
|
495
535
|
}
|
|
496
536
|
if (attempt < this.maxRetries) {
|
|
497
|
-
const delayMs = error
|
|
537
|
+
const delayMs = isMembraneError(error) && typeof error.retryAfterMs === 'number'
|
|
498
538
|
? error.retryAfterMs
|
|
499
539
|
: Math.pow(2, attempt) * 1000;
|
|
500
540
|
return { retry: true, delayMs };
|
|
@@ -513,10 +553,32 @@ const DEFAULT_MAINTENANCE_INTERVAL_MS = 5000;
|
|
|
513
553
|
/** Bound one pass so a large backlog yields to inference and other agents. */
|
|
514
554
|
const MAINTENANCE_TICKS_PER_PASS = 8;
|
|
515
555
|
const PROVIDER_ACCELERATION_DEFAULT_COOLDOWN_MS = 65_000;
|
|
556
|
+
/** Cap on waits the framework chooses itself. A provider's stated wait is
|
|
557
|
+
* never capped: it is a lower bound (see ProviderWaits). It is also the
|
|
558
|
+
* longest stated wait an ephemeral run sleeps in its turn: a longer one ends
|
|
559
|
+
* the run instead (nonOwningProviderWait). */
|
|
516
560
|
const PROVIDER_ACCELERATION_MAX_COOLDOWN_MS = 10 * 60_000;
|
|
517
561
|
const PROVIDER_ACCELERATION_JITTER_MS = 5_000;
|
|
562
|
+
/** Longest delay one setTimeout can represent (2^31-1 ms, about 24.8 days);
|
|
563
|
+
* a longer timer fires almost at once, so longer waits re-arm in steps. */
|
|
564
|
+
const MAX_TIMER_DELAY_MS = 2_147_483_647;
|
|
565
|
+
/** A stated wait recorded to hold a model longer than this, or until
|
|
566
|
+
* released, raises an ops alert. It is honoured in full either way: a hold
|
|
567
|
+
* that long is a quota-scale event or a provider's mistake, and a person
|
|
568
|
+
* decides which (release-provider-wait). Shorter waits are pacing. */
|
|
569
|
+
const PROVIDER_WAIT_ALERT_MS = 60 * 60_000;
|
|
570
|
+
/**
|
|
571
|
+
* A MembraneError, from this framework's copy of membrane or from another: the
|
|
572
|
+
* host supplies the Membrane instance (config.membrane), so with two copies
|
|
573
|
+
* installed its errors are instances of a class this framework's `instanceof`
|
|
574
|
+
* doesn't match. Recognised by name as well, as context-manager's errors are;
|
|
575
|
+
* its fields are the same, so callers still check each one's type.
|
|
576
|
+
*/
|
|
577
|
+
function isMembraneError(error) {
|
|
578
|
+
return error instanceof MembraneError || (error instanceof Error && error.name === 'MembraneError');
|
|
579
|
+
}
|
|
518
580
|
function isOrganizationAccelerationRateLimit(error) {
|
|
519
|
-
if (!(error
|
|
581
|
+
if (!isMembraneError(error) || error.type !== 'rate_limit')
|
|
520
582
|
return false;
|
|
521
583
|
return (/organization(?:'s)?[^\n]{0,120}maximum(?:\s+usage)?\s+increase\s+rate/i.test(error.message) ||
|
|
522
584
|
/organization(?:'s)?[^\n]{0,120}acceleration(?:\s+limit)?/i.test(error.message));
|
|
@@ -552,11 +614,267 @@ function extractGateFields(event) {
|
|
|
552
614
|
: undefined;
|
|
553
615
|
return { content, metadata, mount, paths };
|
|
554
616
|
}
|
|
555
|
-
/**
|
|
556
|
-
*
|
|
557
|
-
*
|
|
558
|
-
|
|
559
|
-
|
|
617
|
+
/**
|
|
618
|
+
* Longest failure text the inference-failure pipeline writes to one sink: the
|
|
619
|
+
* reason in the exhaustion stderr line, a field of a logs/failures.log record,
|
|
620
|
+
* an ops alert's message, the inference log's `error`, and a trace event's
|
|
621
|
+
* `error` and `stack`. A failure's text is whatever its source put in
|
|
622
|
+
* `Error.message`, and a provider can echo the entire rejected request there
|
|
623
|
+
* (one OpenAI-compatible 400 echoed ~1.7 MB). The sinks with shorter bounds
|
|
624
|
+
* name them below, and every cut goes through boundFailureText. This bound
|
|
625
|
+
* doesn't depend on membrane bounding its own errors.
|
|
626
|
+
*/
|
|
627
|
+
const FAILURE_TEXT_MAX_CHARS = 2_000;
|
|
628
|
+
/** Longest failure reason written into an agent's own context (the
|
|
629
|
+
* `[inference-failed]` marker): enough for a provider's explanation, not a
|
|
630
|
+
* self-dump the agent then carries in every later request. */
|
|
631
|
+
const FAILURE_MARKER_REASON_MAX_CHARS = 600;
|
|
632
|
+
/** Longest failure reason in the rewind marker, also the agent's own context:
|
|
633
|
+
* enough to identify the failure class without a wall of provider JSON. */
|
|
634
|
+
const REWIND_MARKER_REASON_MAX_CHARS = 160;
|
|
635
|
+
/** Longest failure reason in an agent's status (`lastInference.lastError`) and
|
|
636
|
+
* in the compile-refused ops alert. */
|
|
637
|
+
const FAILURE_SUMMARY_MAX_CHARS = 300;
|
|
638
|
+
/** Longest alert message posted to the ops webhook. */
|
|
639
|
+
const OPS_WEBHOOK_MESSAGE_MAX_CHARS = 500;
|
|
640
|
+
/** Longest provider error code recorded verbatim; real codes are short tokens.
|
|
641
|
+
* A failure's name and type are bounded the same way. */
|
|
642
|
+
const PROVIDER_ERROR_CODE_MAX_CHARS = 128;
|
|
643
|
+
const isHighSurrogate = (text, index) => {
|
|
644
|
+
const code = text.charCodeAt(index);
|
|
645
|
+
return code >= 0xd800 && code <= 0xdbff;
|
|
646
|
+
};
|
|
647
|
+
const isLowSurrogate = (text, index) => {
|
|
648
|
+
const code = text.charCodeAt(index);
|
|
649
|
+
return code >= 0xdc00 && code <= 0xdfff;
|
|
650
|
+
};
|
|
651
|
+
/** Whether a cut before `index` would separate the halves of a surrogate pair. */
|
|
652
|
+
const splitsPair = (text, index) => index > 0 && index < text.length && isHighSurrogate(text, index - 1) && isLowSurrogate(text, index);
|
|
653
|
+
/**
|
|
654
|
+
* `text` when it fits within `max` characters; otherwise its head and tail
|
|
655
|
+
* around a marker stating how many of how many characters were omitted, the
|
|
656
|
+
* whole within `max`. A cut that would separate a surrogate pair moves toward
|
|
657
|
+
* omission, so the result never holds half of a pair the text held whole, and
|
|
658
|
+
* bounding bounded text returns it as is. Each sink cuts the whole text, so
|
|
659
|
+
* its marker states the whole length.
|
|
660
|
+
*/
|
|
661
|
+
function boundFailureText(text, max = FAILURE_TEXT_MAX_CHARS) {
|
|
662
|
+
if (text.length <= max)
|
|
663
|
+
return text;
|
|
664
|
+
const markerLength = ` …[${text.length} of ${text.length} characters omitted]… `.length;
|
|
665
|
+
const budget = Math.max(0, max - markerLength);
|
|
666
|
+
const tailLength = Math.floor(budget / 4);
|
|
667
|
+
let headEnd = budget - tailLength;
|
|
668
|
+
if (splitsPair(text, headEnd))
|
|
669
|
+
headEnd--;
|
|
670
|
+
let tailStart = text.length - tailLength;
|
|
671
|
+
if (splitsPair(text, tailStart))
|
|
672
|
+
tailStart++;
|
|
673
|
+
return `${text.slice(0, headEnd)} …[${tailStart - headEnd} of ${text.length} characters omitted]… ${text.slice(tailStart)}`;
|
|
674
|
+
}
|
|
675
|
+
/** A provider error code, verbatim when token-sized; otherwise bounded. */
|
|
676
|
+
function boundProviderErrorCode(code) {
|
|
677
|
+
if (typeof code !== 'string')
|
|
678
|
+
return undefined;
|
|
679
|
+
return boundFailureText(code, PROVIDER_ERROR_CODE_MAX_CHARS);
|
|
680
|
+
}
|
|
681
|
+
/**
|
|
682
|
+
* A value's JSON for a failure sink, with every Error inside it written as its
|
|
683
|
+
* whole projection (failureDescription): serialising an Error writes its own
|
|
684
|
+
* fields, a MembraneError's whole `rawRequest` among them. The projection is
|
|
685
|
+
* left uncut, as a nested string is, so the sink's one cut of the JSON states
|
|
686
|
+
* its whole length. An Error is recognised by instanceof, so one from another
|
|
687
|
+
* realm (a vm context) is written as its own JSON, as any object is. Throws,
|
|
688
|
+
* or returns undefined, as JSON.stringify does.
|
|
689
|
+
*/
|
|
690
|
+
function failureJson(value) {
|
|
691
|
+
return JSON.stringify(value, (_key, nested) => (nested instanceof Error ? failureDescription(nested) : nested));
|
|
692
|
+
}
|
|
693
|
+
/**
|
|
694
|
+
* The whole text describeFailure cuts. For an Error, its head (name,
|
|
695
|
+
* classification, status, provider code, each bounded) and its whole
|
|
696
|
+
* message. For a primitive, its String. For another object, its String when
|
|
697
|
+
* that's text of its own (a Buffer, an Error from another realm, a class
|
|
698
|
+
* with its own toString). An array, or an object whose String is empty or
|
|
699
|
+
* only its tag, such as `[object Object]`, is its JSON instead
|
|
700
|
+
* (failureJson), so its fields and elements are kept; its tag stands when
|
|
701
|
+
* JSON can't write it or finds nothing in it.
|
|
702
|
+
*/
|
|
703
|
+
function failureDescription(error) {
|
|
704
|
+
try {
|
|
705
|
+
if (!(error instanceof Error)) {
|
|
706
|
+
if (typeof error !== 'object' || error === null)
|
|
707
|
+
return String(error);
|
|
708
|
+
if (!Array.isArray(error)) {
|
|
709
|
+
let text;
|
|
710
|
+
try {
|
|
711
|
+
text = String(error);
|
|
712
|
+
}
|
|
713
|
+
catch { /* no String of its own: its JSON, below */ }
|
|
714
|
+
if (text && !/^\[object [^\]]*\]$/.test(text))
|
|
715
|
+
return text;
|
|
716
|
+
}
|
|
717
|
+
try {
|
|
718
|
+
const json = failureJson(error);
|
|
719
|
+
if (json !== undefined && json !== '{}')
|
|
720
|
+
return json;
|
|
721
|
+
}
|
|
722
|
+
catch { /* its tag, below */ }
|
|
723
|
+
return Object.prototype.toString.call(error);
|
|
724
|
+
}
|
|
725
|
+
const fields = error;
|
|
726
|
+
// Every component is remote or caller-supplied text, the name included.
|
|
727
|
+
const classification = [
|
|
728
|
+
typeof fields.type === 'string' ? boundFailureText(fields.type, PROVIDER_ERROR_CODE_MAX_CHARS) : undefined,
|
|
729
|
+
typeof fields.httpStatus === 'number' ? `HTTP ${fields.httpStatus}` : undefined,
|
|
730
|
+
boundProviderErrorCode(fields.providerErrorCode),
|
|
731
|
+
typeof fields.retryable === 'boolean' ? `retryable=${fields.retryable}` : undefined,
|
|
732
|
+
].filter((part) => part !== undefined);
|
|
733
|
+
const name = boundFailureText(String(error.name), PROVIDER_ERROR_CODE_MAX_CHARS);
|
|
734
|
+
const head = classification.length > 0 ? `${name} (${classification.join(', ')})` : name;
|
|
735
|
+
return `${head}: ${String(error.message)}`;
|
|
736
|
+
}
|
|
737
|
+
catch {
|
|
738
|
+
return '[failure could not be described]';
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
/**
|
|
742
|
+
* A bounded string describing a failure, for logs that would otherwise print
|
|
743
|
+
* the error object; it is one line unless one of its parts has newlines, as a
|
|
744
|
+
* message can. Printing the object prints its message whole, in the stack,
|
|
745
|
+
* and every own field: a MembraneError's `rawRequest` is the whole request
|
|
746
|
+
* that failed, and Node's console prints each of its strings, a long system
|
|
747
|
+
* prompt among them, up to 10,000 characters. The projection keeps what a
|
|
748
|
+
* reader needs (name, classification, status, provider code, message) and
|
|
749
|
+
* nothing else. The head's parts are bounded each, and the whole
|
|
750
|
+
* (failureDescription) is cut once, so its marker states the length of the
|
|
751
|
+
* head and the whole message.
|
|
752
|
+
*/
|
|
753
|
+
function describeFailure(error) {
|
|
754
|
+
return boundFailureText(failureDescription(error));
|
|
755
|
+
}
|
|
756
|
+
/** A stack frame's line: an indent of spaces or tabs, then `at `. The indent
|
|
757
|
+
* never crosses a line break, so a run of blank lines isn't rescanned from
|
|
758
|
+
* each line's start. */
|
|
759
|
+
const FRAME_LINE = /^[ \t]+at /m;
|
|
760
|
+
/**
|
|
761
|
+
* describeFailure, then the error's stack frames, bounded, on the lines
|
|
762
|
+
* below: for a log of a failure the framework's own code may have thrown,
|
|
763
|
+
* where the frames say where. The stack opens with the message, and a
|
|
764
|
+
* message's own lines can look like frames (an echoed request quoting a stack
|
|
765
|
+
* trace), so when the message is the stack's header, before its first frame
|
|
766
|
+
* line, the frames are the stack from its first frame line after the
|
|
767
|
+
* message. Otherwise (a stack without its header, a message that isn't in
|
|
768
|
+
* the stack or is empty) they are the stack from its first frame line. The
|
|
769
|
+
* leak the projection closes was the object's own fields, not its stack.
|
|
770
|
+
*/
|
|
771
|
+
function describeFailureWithFrames(error) {
|
|
772
|
+
let frames;
|
|
773
|
+
try {
|
|
774
|
+
const stack = error instanceof Error ? error.stack : undefined;
|
|
775
|
+
if (typeof stack === 'string') {
|
|
776
|
+
const message = String(error.message);
|
|
777
|
+
// The message is the stack's header only before its first frame: a
|
|
778
|
+
// stack without its header (copied, or frames only) can hold the
|
|
779
|
+
// message's text inside a frame's name.
|
|
780
|
+
const firstFrame = stack.search(FRAME_LINE);
|
|
781
|
+
const at = message ? stack.indexOf(message) : -1;
|
|
782
|
+
const frameLine = new RegExp(FRAME_LINE.source, 'gm');
|
|
783
|
+
frameLine.lastIndex = at >= 0 && (firstFrame < 0 || at < firstFrame) ? at + message.length : 0;
|
|
784
|
+
const found = frameLine.exec(stack);
|
|
785
|
+
if (found)
|
|
786
|
+
frames = boundFailureText(stack.slice(found.index));
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
catch { /* the projection alone */ }
|
|
790
|
+
const described = describeFailure(error);
|
|
791
|
+
return frames === undefined ? described : `${described}\n${frames}`;
|
|
792
|
+
}
|
|
793
|
+
/**
|
|
794
|
+
* A failure record's fields, each bounded, for logs/failures.log and the
|
|
795
|
+
* ops:alert trace: a string to FAILURE_TEXT_MAX_CHARS; an Error to its
|
|
796
|
+
* projection, since serialising one writes its own fields, a MembraneError's
|
|
797
|
+
* whole `rawRequest` among them; any other value to its JSON, with each Error
|
|
798
|
+
* inside it as its projection (failureJson), as the plain data that JSON
|
|
799
|
+
* describes when it fits FAILURE_TEXT_MAX_CHARS and as the JSON text,
|
|
800
|
+
* bounded, when it doesn't. So the trace's data is what failures.log writes:
|
|
801
|
+
* a Map or an object with toJSON whose JSON is small never hands a listener
|
|
802
|
+
* the value itself.
|
|
803
|
+
*/
|
|
804
|
+
function boundFailureFields(record) {
|
|
805
|
+
const bounded = {};
|
|
806
|
+
for (const [key, value] of Object.entries(record)) {
|
|
807
|
+
if (typeof value === 'string')
|
|
808
|
+
bounded[key] = boundFailureText(value);
|
|
809
|
+
else if (value instanceof Error)
|
|
810
|
+
bounded[key] = describeFailure(value);
|
|
811
|
+
else {
|
|
812
|
+
let json;
|
|
813
|
+
try {
|
|
814
|
+
json = failureJson(value);
|
|
815
|
+
}
|
|
816
|
+
catch {
|
|
817
|
+
bounded[key] = describeFailure(value);
|
|
818
|
+
continue;
|
|
819
|
+
}
|
|
820
|
+
// A value JSON writes nothing for (a function, undefined) is left out,
|
|
821
|
+
// as the log's JSON leaves it out.
|
|
822
|
+
if (json === undefined)
|
|
823
|
+
continue;
|
|
824
|
+
bounded[key] = json.length <= FAILURE_TEXT_MAX_CHARS ? JSON.parse(json) : boundFailureText(json);
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
return bounded;
|
|
828
|
+
}
|
|
829
|
+
/**
|
|
830
|
+
* Whether the poison-history breaker may shed the newest exchange for a
|
|
831
|
+
* failure: an `invalid_request` at HTTP 400, 413 or 422 or with no status, or
|
|
832
|
+
* a `context_length` at 413. The status is a heuristic for when a shed may
|
|
833
|
+
* help, not proof that the history is at fault.
|
|
834
|
+
*
|
|
835
|
+
* 400 and 422 are how a provider answers a request that fails its
|
|
836
|
+
* validation, as a history it can't accept does, but other refusals come
|
|
837
|
+
* back the same way, such as Anthropic's 400 for a credit balance that's too
|
|
838
|
+
* low. A 413 is the request's size. When the newest exchange made it too
|
|
839
|
+
* large, as an oversized attachment does, shedding that exchange may bring
|
|
840
|
+
* the request back under the limit, and nothing else in the framework
|
|
841
|
+
* recovers a 413; when the history as a whole is too large, the shed doesn't
|
|
842
|
+
* help, and the breaker stops at its rewind cap. The 413 arm takes both
|
|
843
|
+
* labels a 413 arrives under, invalid_request and context_length, because
|
|
844
|
+
* membrane's label for a 413 differs by version and adapter. Other statuses
|
|
845
|
+
* keep the history, since shed history can't be got back and the status
|
|
846
|
+
* gives no sign that a shed would help: a 404 is a model or an endpoint that
|
|
847
|
+
* doesn't exist, and 405, 409, 410 and 415 are about the HTTP request or the
|
|
848
|
+
* resource, as a rule, rather than the conversation.
|
|
849
|
+
*
|
|
850
|
+
* The table test in test/inference-exhaustion-rewind.test.ts is this
|
|
851
|
+
* predicate's contract. The notes below on which membrane version gives which
|
|
852
|
+
* label are its history, and will go stale as hosts move to newer releases.
|
|
853
|
+
*
|
|
854
|
+
* Which failures reach the gate with which label depends on the membrane:
|
|
855
|
+
* - As membrane 0.5.86 classifies them: Anthropic files a 400 as
|
|
856
|
+
* invalid_request (after its context check) and its other 4xx as unknown.
|
|
857
|
+
* The chat adapters never file one as invalid_request, since they classify
|
|
858
|
+
* by the error's message text, and a context_length they file that way
|
|
859
|
+
* carries status 400 whatever the response's was. The raw-fetch Responses
|
|
860
|
+
* adapter files every status it doesn't list (404, 405, 409, 410, 413,
|
|
861
|
+
* 415, 422 …) as invalid_request, so it alone produces an invalid_request
|
|
862
|
+
* at 413 or 422, and no 0.5.86 adapter produces a context_length at 413.
|
|
863
|
+
* - From membrane #56 (merged, not yet in a release), one table serves every
|
|
864
|
+
* adapter. Auth codes and context overflow are checked on a 400 or 422
|
|
865
|
+
* first, a 404 is invalid_request, a 413 is context_length, and other 4xx
|
|
866
|
+
* are invalid_request. So a 413 still sheds, an Anthropic one included,
|
|
867
|
+
* while a context_length at 400 or 422, a token overflow, keeps the
|
|
868
|
+
* history. The 422 arm then also covers an OpenAI-compatible server, such
|
|
869
|
+
* as a FastAPI one, that answers a validation failure with 422.
|
|
870
|
+
*
|
|
871
|
+
* An error with no status, from a membrane that reports none, keeps the rule
|
|
872
|
+
* the breaker had before: the type alone.
|
|
873
|
+
*/
|
|
874
|
+
export function shedMayHelp(errorType, httpStatus) {
|
|
875
|
+
if (httpStatus === 413)
|
|
876
|
+
return errorType === 'invalid_request' || errorType === 'context_length';
|
|
877
|
+
return errorType === 'invalid_request' && (httpStatus === undefined || httpStatus === 400 || httpStatus === 422);
|
|
560
878
|
}
|
|
561
879
|
/** The most of a thrown value's or a tool error's text that reaches a log line or a server's error answer. */
|
|
562
880
|
const THROWN_TEXT_LIMIT = 500;
|
|
@@ -688,6 +1006,10 @@ export class AgentFramework {
|
|
|
688
1006
|
providerAccelerationDefaultCooldownMs = PROVIDER_ACCELERATION_DEFAULT_COOLDOWN_MS;
|
|
689
1007
|
providerAccelerationJitterMs = PROVIDER_ACCELERATION_JITTER_MS;
|
|
690
1008
|
providerHoldHook;
|
|
1009
|
+
/** Provider waits (stated retry-after) per (agent, model), kept across restarts. */
|
|
1010
|
+
providerWaits;
|
|
1011
|
+
/** Wakers for error-policy retry waits in progress: stop() ends them. */
|
|
1012
|
+
retryWaitWakers = new Set();
|
|
691
1013
|
providerAdmissionClosed = false;
|
|
692
1014
|
/** Invalidates framework turns still assembling when stop begins. */
|
|
693
1015
|
lifecycleGeneration = 0;
|
|
@@ -1069,6 +1391,13 @@ export class AgentFramework {
|
|
|
1069
1391
|
this.operatorLog = operatorLog;
|
|
1070
1392
|
this.store = store;
|
|
1071
1393
|
this.ownsStore = ownsStore;
|
|
1394
|
+
// Unreadable recorded waits hold every agent's provider calls, and a
|
|
1395
|
+
// persistent cause never reads again: a person has to hear of it.
|
|
1396
|
+
this.providerWaits = new ProviderWaits(store, {
|
|
1397
|
+
onUnreadable: (error) => this.opsAlert('provider-wait', 'framework', `recorded provider waits could not be read (${error}): no provider call is admitted for any agent ` +
|
|
1398
|
+
'until they read again (re-read every 30 s) or an operator releases an agent\'s waits with ' +
|
|
1399
|
+
'release-provider-wait; such a release lasts for this process only', { data: { model: EVERY_MODEL, until: null } }),
|
|
1400
|
+
});
|
|
1072
1401
|
this.membrane = membrane;
|
|
1073
1402
|
this.inferencePolicy = inferencePolicy;
|
|
1074
1403
|
this.errorPolicy = errorPolicy;
|
|
@@ -1085,14 +1414,16 @@ export class AgentFramework {
|
|
|
1085
1414
|
emitTrace: (e) => this.emitTrace({ ...e }),
|
|
1086
1415
|
});
|
|
1087
1416
|
// Initialize module registry with callbacks
|
|
1088
|
-
|
|
1417
|
+
// A module's events, through pushEvent() or its deprecated queue handle,
|
|
1418
|
+
// go through pushModuleEvent(), which names the module.
|
|
1419
|
+
this.moduleRegistry = new ModuleRegistry(store, { push: (event, moduleName) => this.pushModuleEvent(event, moduleName, false) }, {
|
|
1089
1420
|
getAgents: () => Array.from(this.agents.values()),
|
|
1090
1421
|
addMessage: (p, c, m) => this.addMessage(p, c, m),
|
|
1091
1422
|
editMessage: (id, c) => this.editMessage(id, c),
|
|
1092
1423
|
removeMessage: (id) => this.removeMessage(id),
|
|
1093
1424
|
getMessage: (id) => this.getMessage(id),
|
|
1094
1425
|
queryMessages: (filter) => this.queryMessages(filter),
|
|
1095
|
-
pushEvent: (event) => this.
|
|
1426
|
+
pushEvent: (event, moduleName) => this.pushModuleEvent(event, moduleName, true),
|
|
1096
1427
|
onTrace: (listener) => this.onTrace(listener),
|
|
1097
1428
|
// The ONLY entry that confers module origin — a private closure the
|
|
1098
1429
|
// framework hands to its own module registry. The public
|
|
@@ -1565,6 +1896,8 @@ export class AgentFramework {
|
|
|
1565
1896
|
// store is still open, rather than leaving them to a reboot replay.
|
|
1566
1897
|
this.ackDeferredWrites();
|
|
1567
1898
|
this.providerAdmissionClosed = true;
|
|
1899
|
+
for (const wake of [...this.retryWaitWakers])
|
|
1900
|
+
wake();
|
|
1568
1901
|
this.lifecycleGeneration++;
|
|
1569
1902
|
// The generation this stop ends; withheld spills close for it alone, so a
|
|
1570
1903
|
// start() made before this stop finishes reopens them for its own.
|
|
@@ -1716,7 +2049,13 @@ export class AgentFramework {
|
|
|
1716
2049
|
providerGateBlocked(agentName) {
|
|
1717
2050
|
const gate = this.providerGates.get(agentName);
|
|
1718
2051
|
return (gate?.primaryDepth ?? 0) > 0 || (gate?.primaryPending ?? false) ||
|
|
1719
|
-
this.
|
|
2052
|
+
this.cooldownBlocksAuxiliary(agentName);
|
|
2053
|
+
}
|
|
2054
|
+
/** Acceleration and host holds park the whole agent; a provider wait holds
|
|
2055
|
+
* only its own model, which auxiliary admission checks per call. */
|
|
2056
|
+
cooldownBlocksAuxiliary(agentName) {
|
|
2057
|
+
const cooldown = this.providerAccelerationCooldowns.get(agentName);
|
|
2058
|
+
return cooldown !== undefined && cooldown.waitModel === undefined;
|
|
1720
2059
|
}
|
|
1721
2060
|
acquirePrimaryProviderGate(agentName) {
|
|
1722
2061
|
const gate = this.providerGate(agentName);
|
|
@@ -1730,7 +2069,7 @@ export class AgentFramework {
|
|
|
1730
2069
|
}
|
|
1731
2070
|
flushAuxiliaryAdmission(agentName) {
|
|
1732
2071
|
const gate = this.providerGate(agentName);
|
|
1733
|
-
if (gate.primaryDepth > 0 || gate.primaryPending || this.
|
|
2072
|
+
if (gate.primaryDepth > 0 || gate.primaryPending || this.cooldownBlocksAuxiliary(agentName))
|
|
1734
2073
|
return;
|
|
1735
2074
|
for (const resolve of gate.auxiliaryWaiters.splice(0))
|
|
1736
2075
|
resolve();
|
|
@@ -1741,8 +2080,9 @@ export class AgentFramework {
|
|
|
1741
2080
|
return;
|
|
1742
2081
|
await new Promise((resolve) => gate.idleWaiters.push(resolve));
|
|
1743
2082
|
}
|
|
1744
|
-
async withAuxiliaryAdmission(agentName, run) {
|
|
2083
|
+
async withAuxiliaryAdmission(agentName, run, model) {
|
|
1745
2084
|
const gate = this.providerGate(agentName);
|
|
2085
|
+
this.refuseHeldProviderModel(agentName, model);
|
|
1746
2086
|
if (this.providerGateBlocked(agentName)) {
|
|
1747
2087
|
gate.deferredAuxiliary++;
|
|
1748
2088
|
try {
|
|
@@ -1755,17 +2095,23 @@ export class AgentFramework {
|
|
|
1755
2095
|
if (this.providerAdmissionClosed) {
|
|
1756
2096
|
throw new Error(`Provider admission closed while stopping (${agentName})`);
|
|
1757
2097
|
}
|
|
2098
|
+
// A wait recorded while this call was parked binds it too.
|
|
2099
|
+
this.refuseHeldProviderModel(agentName, model);
|
|
1758
2100
|
gate.auxiliaryInFlight++;
|
|
1759
2101
|
try {
|
|
1760
2102
|
return await run();
|
|
1761
2103
|
}
|
|
1762
2104
|
catch (error) {
|
|
1763
2105
|
// Compression can be the first call to find a spent quota (idle agent,
|
|
1764
|
-
// scheduled maintenance).
|
|
1765
|
-
//
|
|
2106
|
+
// scheduled maintenance). Record a stated wait for this call's model,
|
|
2107
|
+
// and arm the host hold so later passes wait at the gate instead of
|
|
2108
|
+
// retrying into it; this pass still fails to its caller.
|
|
1766
2109
|
const agent = this.agents.get(agentName);
|
|
1767
|
-
if (agent && error instanceof Error)
|
|
2110
|
+
if (agent && error instanceof Error) {
|
|
2111
|
+
if (model !== undefined)
|
|
2112
|
+
this.recordProviderWait(agentName, model, error);
|
|
1768
2113
|
this.holdProviderAcceleration(agent, error, undefined, true);
|
|
2114
|
+
}
|
|
1769
2115
|
throw error;
|
|
1770
2116
|
}
|
|
1771
2117
|
finally {
|
|
@@ -1776,22 +2122,62 @@ export class AgentFramework {
|
|
|
1776
2122
|
}
|
|
1777
2123
|
}
|
|
1778
2124
|
auxiliaryMembraneFor(agentName) {
|
|
2125
|
+
return this.membraneAdmitting((run, model) => this.withAuxiliaryAdmission(agentName, run, model));
|
|
2126
|
+
}
|
|
2127
|
+
/**
|
|
2128
|
+
* A conversation agent's auxiliary calls (its context manager's
|
|
2129
|
+
* compression). A conversation agent takes no provider gate: admission
|
|
2130
|
+
* belongs to the persistent resident, and a generation-unique fork must not
|
|
2131
|
+
* leave a gate behind. But its stated waits are recorded for (agent, model)
|
|
2132
|
+
* as a resident's are, so they bind its auxiliary calls too: a held model
|
|
2133
|
+
* is refused without a call, and an auxiliary call's own stated wait is
|
|
2134
|
+
* recorded for it. (An ephemeral run records nothing, its name being
|
|
2135
|
+
* single-use, so its context manager keeps the plain membrane.)
|
|
2136
|
+
*/
|
|
2137
|
+
conversationAuxiliaryMembraneFor(agentName) {
|
|
2138
|
+
return this.membraneAdmitting(async (run, model) => {
|
|
2139
|
+
this.refuseHeldProviderModel(agentName, model);
|
|
2140
|
+
try {
|
|
2141
|
+
return await run();
|
|
2142
|
+
}
|
|
2143
|
+
catch (error) {
|
|
2144
|
+
if (model !== undefined && error instanceof Error)
|
|
2145
|
+
this.recordProviderWait(agentName, model, error);
|
|
2146
|
+
throw error;
|
|
2147
|
+
}
|
|
2148
|
+
});
|
|
2149
|
+
}
|
|
2150
|
+
/** The framework's membrane, with each complete() call passed through
|
|
2151
|
+
* `admit` along with the model it names. */
|
|
2152
|
+
membraneAdmitting(admit) {
|
|
1779
2153
|
const target = this.membrane;
|
|
1780
2154
|
return new Proxy(target, { get: (obj, prop, receiver) => {
|
|
1781
2155
|
const value = Reflect.get(obj, prop, receiver);
|
|
1782
2156
|
if (prop === 'complete' && typeof value === 'function') {
|
|
1783
|
-
return (...args) =>
|
|
2157
|
+
return (...args) => {
|
|
2158
|
+
const model = args[0]?.config?.model;
|
|
2159
|
+
return admit(() => Reflect.apply(value, this.membrane, args), typeof model === 'string' ? model : undefined);
|
|
2160
|
+
};
|
|
1784
2161
|
}
|
|
1785
2162
|
return typeof value === 'function' ? value.bind(this.membrane) : value;
|
|
1786
2163
|
} });
|
|
1787
2164
|
}
|
|
2165
|
+
/** An acceleration hold's length: AF's own pacing (`ownMs`: its default
|
|
2166
|
+
* plus jitter, capped), or the provider's stated wait in its place
|
|
2167
|
+
* (`stated`). The stated wait is a lower bound: jitter only adds to it,
|
|
2168
|
+
* and the cap applies only to AF's own default. A negative stated wait is
|
|
2169
|
+
* a stated 0, as in ProviderWaits: the 1 s floor, plus jitter. */
|
|
1788
2170
|
accelerationCooldownMs(agentName, error) {
|
|
1789
|
-
const base = error.retryAfterMs ?? this.providerAccelerationDefaultCooldownMs;
|
|
1790
2171
|
let hash = 0;
|
|
1791
2172
|
for (const ch of agentName)
|
|
1792
2173
|
hash = ((hash * 31) + ch.charCodeAt(0)) >>> 0;
|
|
1793
2174
|
const jitter = this.providerAccelerationJitterMs > 0 ? hash % (this.providerAccelerationJitterMs + 1) : 0;
|
|
1794
|
-
|
|
2175
|
+
const ownMs = Math.min(PROVIDER_ACCELERATION_MAX_COOLDOWN_MS, Math.max(1_000, this.providerAccelerationDefaultCooldownMs) + jitter);
|
|
2176
|
+
const stated = error.retryAfterMs;
|
|
2177
|
+
if (typeof stated === 'number' && Number.isFinite(stated)) {
|
|
2178
|
+
return { delayMs: Math.max(1_000, stated) + jitter, ownMs, stated: true };
|
|
2179
|
+
}
|
|
2180
|
+
return { delayMs: ownMs, ownMs, stated: false };
|
|
1795
2181
|
}
|
|
1796
2182
|
sameInferenceRequest(a, b) {
|
|
1797
2183
|
return a.agentName === b.agentName && a.reason === b.reason && a.source === b.source &&
|
|
@@ -1804,11 +2190,15 @@ export class AgentFramework {
|
|
|
1804
2190
|
const hold = this.providerHoldHook(error, agentName, { model: this.agents.get(agentName)?.model });
|
|
1805
2191
|
if (!hold || !Number.isFinite(hold.holdMs) || hold.holdMs <= 0)
|
|
1806
2192
|
return undefined;
|
|
2193
|
+
// The reason is printed, kept in the cooldown and shown in the status
|
|
2194
|
+
// snapshot (cooldownReason); without the host's own, it is the error's
|
|
2195
|
+
// message, which can echo the whole request.
|
|
1807
2196
|
return { holdMs: Math.min(PROVIDER_ACCELERATION_MAX_COOLDOWN_MS, Math.max(1_000, hold.holdMs)),
|
|
1808
|
-
reason: typeof hold.reason === 'string' && hold.reason ? hold.reason : error.message };
|
|
2197
|
+
reason: boundFailureText(typeof hold.reason === 'string' && hold.reason ? hold.reason : String(error.message)) };
|
|
1809
2198
|
}
|
|
1810
2199
|
catch (err) {
|
|
1811
|
-
|
|
2200
|
+
// The hook is handed the model's error, and may throw it back.
|
|
2201
|
+
console.error(`[provider-cooldown] providerHold hook threw for ${agentName}; treating as no hold: ${describeFailureWithFrames(err)}`);
|
|
1812
2202
|
return undefined;
|
|
1813
2203
|
}
|
|
1814
2204
|
}
|
|
@@ -1822,7 +2212,7 @@ export class AgentFramework {
|
|
|
1822
2212
|
return false;
|
|
1823
2213
|
clearTimeout(cooldown.timer);
|
|
1824
2214
|
cooldown.until = Date.now() + hold.holdMs;
|
|
1825
|
-
cooldown.timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agentName), hold.holdMs);
|
|
2215
|
+
cooldown.timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agentName), Math.min(hold.holdMs, MAX_TIMER_DELAY_MS));
|
|
1826
2216
|
cooldown.timer.unref?.();
|
|
1827
2217
|
if (hold.reason !== cooldown.reason) {
|
|
1828
2218
|
cooldown.reason = hold.reason;
|
|
@@ -1833,26 +2223,42 @@ export class AgentFramework {
|
|
|
1833
2223
|
holdProviderAcceleration(agent, error, trigger, auxiliary = false) {
|
|
1834
2224
|
if (this.ephemeralRuns.has(agent.name) || this.conversationAgentHomes.has(agent.name))
|
|
1835
2225
|
return false;
|
|
2226
|
+
// A provider's stated wait binds later calls to the primary model, across
|
|
2227
|
+
// restarts. (An auxiliary call records its own model's wait at the gate.)
|
|
2228
|
+
const wait = auxiliary ? undefined : this.recordProviderWait(agent.name, agent.model, error);
|
|
1836
2229
|
// The host knows more than the message wording does: ask it first. An
|
|
1837
2230
|
// auxiliary failure arms host holds only — the built-in acceleration
|
|
1838
2231
|
// cooldown stays a primary-path mechanism.
|
|
1839
2232
|
const hostHold = this.consultProviderHold(error, agent.name);
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
2233
|
+
// A failure the provider says may not be retried keeps its terminal
|
|
2234
|
+
// disposition, whatever its wording (a stated wait still binds after it).
|
|
2235
|
+
const acceleration = !hostHold && !auxiliary && error.retryable !== false
|
|
2236
|
+
&& isOrganizationAccelerationRateLimit(error);
|
|
2237
|
+
// The stated wait binds later calls whatever the failure was. Retaining
|
|
2238
|
+
// and retrying THIS request is for a failure the provider says may be
|
|
2239
|
+
// retried: a non-retryable one keeps its terminal disposition (marker,
|
|
2240
|
+
// failure streak), and the wait applies to the calls that follow it.
|
|
2241
|
+
if (!acceleration && !hostHold) {
|
|
2242
|
+
return wait !== undefined && isMembraneError(error) && error.retryable === true
|
|
2243
|
+
&& this.holdForProviderWait(agent, wait, trigger ? [trigger] : [], true);
|
|
2244
|
+
}
|
|
1843
2245
|
const now = Date.now();
|
|
1844
|
-
const
|
|
2246
|
+
const pacing = hostHold ? undefined : this.accelerationCooldownMs(agent.name, error);
|
|
2247
|
+
const delayMs = hostHold ? hostHold.holdMs : pacing.delayMs;
|
|
1845
2248
|
const existing = this.providerAccelerationCooldowns.get(agent.name);
|
|
1846
2249
|
const held = existing?.heldRequests ?? [];
|
|
1847
2250
|
if (trigger && !held.some((r) => this.sameInferenceRequest(r, trigger)))
|
|
1848
2251
|
held.push(trigger);
|
|
1849
2252
|
if (existing)
|
|
1850
2253
|
clearTimeout(existing.timer);
|
|
1851
|
-
const timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agent.name), delayMs);
|
|
2254
|
+
const timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agent.name), Math.min(delayMs, MAX_TIMER_DELAY_MS));
|
|
1852
2255
|
timer.unref?.();
|
|
1853
2256
|
this.providerAccelerationCooldowns.set(agent.name, { startedAt: existing?.startedAt ?? now,
|
|
1854
|
-
until: now + delayMs, timer, heldRequests: held, reason: hostHold?.reason ?? error.message,
|
|
2257
|
+
until: now + delayMs, timer, heldRequests: held, reason: hostHold?.reason ?? boundFailureText(String(error.message)),
|
|
1855
2258
|
failures: (existing?.failures ?? 0) + 1, ...(hostHold ? { hostHoldError: error } : {}),
|
|
2259
|
+
// The stated wait (recorded above for agent.model) set this hold's
|
|
2260
|
+
// length: an operator release of it returns the hold to AF's own pacing.
|
|
2261
|
+
...(pacing?.stated ? { statedWait: { model: agent.model, ownUntil: now + pacing.ownMs } } : {}),
|
|
1856
2262
|
...(auxiliary && (!existing || existing.auxiliaryOrigin) ? { auxiliaryOrigin: true } : {}) });
|
|
1857
2263
|
if (auxiliary && !existing) {
|
|
1858
2264
|
console.error(`[provider-cooldown] agent=${agent.name} host hold (${hostHold.reason}) from an auxiliary call — ` +
|
|
@@ -1862,20 +2268,101 @@ export class AgentFramework {
|
|
|
1862
2268
|
const gate = this.providerGate(agent.name);
|
|
1863
2269
|
gate.primaryPending = true;
|
|
1864
2270
|
this.providerAccelerationRecoveries.set(agent.name, { startedAt: existing?.startedAt ?? now,
|
|
1865
|
-
failures: (existing?.failures ?? 0) + 1, heldRequests: held.length, reason: error.message });
|
|
2271
|
+
failures: (existing?.failures ?? 0) + 1, heldRequests: held.length, reason: boundFailureText(String(error.message)) });
|
|
1866
2272
|
console.error(`[provider-cooldown] agent=${agent.name} ` +
|
|
1867
2273
|
`${hostHold ? `host hold (${hostHold.reason})` : 'organization acceleration 429'} — ` +
|
|
1868
2274
|
`holding primary/auxiliary for ${delayMs}ms; ${held.length} request(s) retained`);
|
|
1869
2275
|
return true;
|
|
1870
2276
|
}
|
|
2277
|
+
/**
|
|
2278
|
+
* A provider's stated wait on a turn that doesn't own provider admission:
|
|
2279
|
+
* a conversation agent's or an ephemeral run's. Left to the error policy,
|
|
2280
|
+
* which returns a stated wait as it is, a 429 stating two hours would hold
|
|
2281
|
+
* the turn asleep two hours, up to three times, with nothing recorded and
|
|
2282
|
+
* nothing raised (membrane #56 review, M3). So:
|
|
2283
|
+
* - A conversation agent's turns come through the scheduler as a resident's
|
|
2284
|
+
* do, so it treats the wait as a resident does: recorded for (agent,
|
|
2285
|
+
* model), alerted past PROVIDER_WAIT_ALERT_MS, and the turn held until the
|
|
2286
|
+
* wait passes or an operator releases it ('held'). A non-retryable failure
|
|
2287
|
+
* records the wait for the calls after it and keeps its own disposition.
|
|
2288
|
+
* - An ephemeral run's name is single-use, so a wait recorded for it would
|
|
2289
|
+
* bind nothing, and its caller is waiting on it. A retryable stated wait
|
|
2290
|
+
* up to PROVIDER_ACCELERATION_MAX_COOLDOWN_MS is left to the error policy,
|
|
2291
|
+
* slept and retried. A longer one, or one that can't be represented, ends
|
|
2292
|
+
* the run now ('fail'): it is never retried before the provider's time,
|
|
2293
|
+
* and the caller hears why at once.
|
|
2294
|
+
* Anything else is the error policy's.
|
|
2295
|
+
*/
|
|
2296
|
+
nonOwningProviderWait(agent, error, trigger) {
|
|
2297
|
+
// From either copy of membrane (isMembraneError), a wait is stated only
|
|
2298
|
+
// as a number, as the default error policy reads it.
|
|
2299
|
+
if (!isMembraneError(error) || typeof error.retryAfterMs !== 'number')
|
|
2300
|
+
return undefined;
|
|
2301
|
+
if (this.conversationAgentHomes.has(agent.name)) {
|
|
2302
|
+
const wait = this.recordProviderWait(agent.name, agent.model, error);
|
|
2303
|
+
return error.retryable === true && wait !== undefined && this.holdForProviderWait(agent, wait, trigger ? [trigger] : [], true)
|
|
2304
|
+
? 'held'
|
|
2305
|
+
: undefined;
|
|
2306
|
+
}
|
|
2307
|
+
if (this.ephemeralRuns.has(agent.name) && error.retryable === true
|
|
2308
|
+
&& !(error.retryAfterMs <= PROVIDER_ACCELERATION_MAX_COOLDOWN_MS)) {
|
|
2309
|
+
return 'fail';
|
|
2310
|
+
}
|
|
2311
|
+
return undefined;
|
|
2312
|
+
}
|
|
2313
|
+
/** Why an ephemeral run ended instead of sleeping a provider's stated wait,
|
|
2314
|
+
* for its caller. */
|
|
2315
|
+
statedWaitNotSlept(error) {
|
|
2316
|
+
const stated = isMembraneError(error) ? error.retryAfterMs : undefined;
|
|
2317
|
+
const cap = `${PROVIDER_ACCELERATION_MAX_COOLDOWN_MS / 1000} s`;
|
|
2318
|
+
if (typeof stated !== 'number' || Number.isNaN(stated)) {
|
|
2319
|
+
return `${error.message} (the provider's stated wait is not a number, and an ephemeral run sleeps only a ` +
|
|
2320
|
+
`stated wait of ${cap} or less, so it ends now)`;
|
|
2321
|
+
}
|
|
2322
|
+
const asked = Number.isFinite(stated) ? `${Math.ceil(stated / 1000)} s` : 'longer than can be represented';
|
|
2323
|
+
return `${error.message} (the provider asked to wait ${asked}; an ephemeral run doesn't sleep a stated wait ` +
|
|
2324
|
+
`longer than ${cap}, so it ends now)`;
|
|
2325
|
+
}
|
|
2326
|
+
/** When a hold's timer next fires: at its deadline, stepped past what one
|
|
2327
|
+
* timer can hold. A hold with no deadline (a wait held until released, or
|
|
2328
|
+
* the stand-in for unreadable records) asks again as often as unreadable
|
|
2329
|
+
* records are read again: records that read again end it, and nothing
|
|
2330
|
+
* announces that. */
|
|
2331
|
+
providerHoldTimerMs(remaining) {
|
|
2332
|
+
return Number.isFinite(remaining)
|
|
2333
|
+
? Math.min(remaining, MAX_TIMER_DELAY_MS)
|
|
2334
|
+
: this.providerWaits?.retryReadMs ?? MAX_TIMER_DELAY_MS;
|
|
2335
|
+
}
|
|
1871
2336
|
releaseProviderAccelerationCooldown(agentName) {
|
|
1872
2337
|
const cooldown = this.providerAccelerationCooldowns.get(agentName);
|
|
1873
2338
|
if (!cooldown)
|
|
1874
2339
|
return;
|
|
2340
|
+
// A wait hold lasts exactly as long as its wait binds: longer once the
|
|
2341
|
+
// wait is extended, while it is indefinite, and over once no wait binds
|
|
2342
|
+
// (passed, released, or the stand-in for unreadable records lifted because
|
|
2343
|
+
// they read again). A different primary model now is not bound by it.
|
|
2344
|
+
if (cooldown.waitModel !== undefined && this.agents.get(agentName)?.model === cooldown.waitModel) {
|
|
2345
|
+
const wait = this.providerWaits?.active(agentName, cooldown.waitModel);
|
|
2346
|
+
cooldown.until = wait ? (wait.until ?? Number.POSITIVE_INFINITY) : Date.now();
|
|
2347
|
+
}
|
|
2348
|
+
const remaining = cooldown.until - Date.now();
|
|
2349
|
+
if (remaining > 0 && (cooldown.waitModel === undefined || this.agents.get(agentName)?.model === cooldown.waitModel)) {
|
|
2350
|
+
clearTimeout(cooldown.timer);
|
|
2351
|
+
cooldown.timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agentName), this.providerHoldTimerMs(remaining));
|
|
2352
|
+
cooldown.timer.unref?.();
|
|
2353
|
+
return;
|
|
2354
|
+
}
|
|
1875
2355
|
if (this.extendHostProviderHold(agentName, cooldown))
|
|
1876
2356
|
return;
|
|
1877
2357
|
clearTimeout(cooldown.timer);
|
|
1878
2358
|
this.providerAccelerationCooldowns.delete(agentName);
|
|
2359
|
+
if (cooldown.waitModel !== undefined) {
|
|
2360
|
+
const model = this.agents.get(agentName)?.model;
|
|
2361
|
+
console.error(`[provider-wait] agent=${agentName} model=${cooldown.waitModel} ` +
|
|
2362
|
+
(model !== undefined && model !== cooldown.waitModel
|
|
2363
|
+
? `no longer holds primary turns: the primary model is now ${model} (the wait stays recorded)`
|
|
2364
|
+
: `primary hold ended after ${Date.now() - cooldown.startedAt}ms`));
|
|
2365
|
+
}
|
|
1879
2366
|
if (cooldown.auxiliaryOrigin && cooldown.heldRequests.length === 0) {
|
|
1880
2367
|
// Nobody asked for an inference while parked: just reopen the gate.
|
|
1881
2368
|
console.error(`[provider-cooldown] agent=${agentName} auxiliary hold released after ${Date.now() - cooldown.startedAt}ms`);
|
|
@@ -1893,6 +2380,199 @@ export class AgentFramework {
|
|
|
1893
2380
|
console.error(`[provider-cooldown] agent=${agentName} released after ${Date.now() - cooldown.startedAt}ms — ` +
|
|
1894
2381
|
`fresh compile queued with ${requests.length} retained request(s)`);
|
|
1895
2382
|
}
|
|
2383
|
+
/**
|
|
2384
|
+
* Wait an error policy's retry delay, all of it: the delay can be a
|
|
2385
|
+
* provider's retry-after, and a single timer past 2^31-1 ms fires almost at
|
|
2386
|
+
* once. Steps through longer waits; stop() wakes it so a shutting-down
|
|
2387
|
+
* framework is never left with a wait parked for days.
|
|
2388
|
+
*/
|
|
2389
|
+
async waitForRetry(ms) {
|
|
2390
|
+
// An infinite delay waits until stop(), like any wait too long to end; a
|
|
2391
|
+
// delay that isn't a number, or is 0 or less, waits none.
|
|
2392
|
+
let remaining = ms > 0 ? ms : 0;
|
|
2393
|
+
while (remaining > 0 && !this.providerAdmissionClosed) {
|
|
2394
|
+
const step = Math.min(remaining, MAX_TIMER_DELAY_MS);
|
|
2395
|
+
remaining -= step;
|
|
2396
|
+
await new Promise((resolve) => {
|
|
2397
|
+
const wakers = this.retryWaitWakers;
|
|
2398
|
+
const done = () => { clearTimeout(timer); wakers.delete(done); resolve(); };
|
|
2399
|
+
const timer = setTimeout(done, step);
|
|
2400
|
+
wakers.add(done);
|
|
2401
|
+
});
|
|
2402
|
+
}
|
|
2403
|
+
}
|
|
2404
|
+
/**
|
|
2405
|
+
* Record a provider's stated wait (a classified error carrying
|
|
2406
|
+
* `retryAfterMs`) for (agent, model). Returns the wait that binds now,
|
|
2407
|
+
* which is the longer of this one and any outstanding one. A wait that
|
|
2408
|
+
* newly binds past PROVIDER_WAIT_ALERT_MS, or until released, raises an
|
|
2409
|
+
* ops alert.
|
|
2410
|
+
*/
|
|
2411
|
+
recordProviderWait(agentName, model, error) {
|
|
2412
|
+
// From either copy of membrane: the host supplies the Membrane instance,
|
|
2413
|
+
// and a wait is stated only as a number, as the default policy reads it.
|
|
2414
|
+
if (!isMembraneError(error) || typeof error.retryAfterMs !== 'number' || !this.providerWaits)
|
|
2415
|
+
return undefined;
|
|
2416
|
+
// The provider's words, bounded; a line break in them must not start a
|
|
2417
|
+
// line that reads as the framework's own where the reason is logged.
|
|
2418
|
+
const reason = safeSlice(withoutControls(error.message), 0, 300);
|
|
2419
|
+
const { wait, changed } = this.providerWaits.set(agentName, model, error.retryAfterMs, reason);
|
|
2420
|
+
// A wait already over (0, or a negative "retry now") holds nothing to report.
|
|
2421
|
+
const remaining = wait.until === null ? Number.POSITIVE_INFINITY : wait.until - Date.now();
|
|
2422
|
+
if (changed && remaining > 0) {
|
|
2423
|
+
const until = wait.until === null
|
|
2424
|
+
? 'until an operator releases it (the stated wait cannot be represented as an instant)'
|
|
2425
|
+
: `until ${new Date(wait.until).toISOString()}`;
|
|
2426
|
+
const stated = `${error.type}${error.httpStatus !== undefined ? ` HTTP ${error.httpStatus}` : ''}, retry-after ${String(error.retryAfterMs)}ms`;
|
|
2427
|
+
console.error(`[provider-wait] agent=${agentName} model=${model} holds ${until}: ${stated}`);
|
|
2428
|
+
if (remaining > PROVIDER_WAIT_ALERT_MS) {
|
|
2429
|
+
this.opsAlert('provider-wait', agentName, `provider wait on ${model} holds ${until} (${stated}: ${reason}); ` +
|
|
2430
|
+
'if the provider did not mean that, release it with release-provider-wait', { data: { model, until: wait.until === null ? null : new Date(wait.until).toISOString() } });
|
|
2431
|
+
}
|
|
2432
|
+
}
|
|
2433
|
+
return wait;
|
|
2434
|
+
}
|
|
2435
|
+
/**
|
|
2436
|
+
* Hold this agent's primary turns for a provider wait on its model. The
|
|
2437
|
+
* requests are kept and released together, into one fresh compile, when
|
|
2438
|
+
* the wait ends: the same path an acceleration cooldown takes, without
|
|
2439
|
+
* parking auxiliary calls to other models.
|
|
2440
|
+
*/
|
|
2441
|
+
holdForProviderWait(agent, wait, requests, failed = false) {
|
|
2442
|
+
const now = Date.now();
|
|
2443
|
+
const until = wait.until ?? Number.POSITIVE_INFINITY;
|
|
2444
|
+
if (until <= now)
|
|
2445
|
+
return false;
|
|
2446
|
+
const existing = this.providerAccelerationCooldowns.get(agent.name);
|
|
2447
|
+
const held = existing?.heldRequests ?? [];
|
|
2448
|
+
for (const request of requests) {
|
|
2449
|
+
if (!held.some((r) => this.sameInferenceRequest(r, request)))
|
|
2450
|
+
held.push(request);
|
|
2451
|
+
}
|
|
2452
|
+
// An acceleration or host hold already parks the whole agent; its release
|
|
2453
|
+
// re-checks this wait at admission.
|
|
2454
|
+
if (existing && existing.waitModel === undefined)
|
|
2455
|
+
return true;
|
|
2456
|
+
if (existing)
|
|
2457
|
+
clearTimeout(existing.timer);
|
|
2458
|
+
const timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agent.name), this.providerHoldTimerMs(until - now));
|
|
2459
|
+
timer.unref?.();
|
|
2460
|
+
this.providerAccelerationCooldowns.set(agent.name, {
|
|
2461
|
+
startedAt: existing?.startedAt ?? now, until, timer, heldRequests: held,
|
|
2462
|
+
// A failed call counts; a turn arriving at a wait recorded earlier doesn't.
|
|
2463
|
+
reason: wait.reason, failures: (existing?.failures ?? 0) + (failed ? 1 : 0), waitModel: wait.model,
|
|
2464
|
+
});
|
|
2465
|
+
if (!existing) {
|
|
2466
|
+
console.error(`[provider-wait] agent=${agent.name} model=${wait.model} holding primary turns ` +
|
|
2467
|
+
`${wait.until === null ? 'until released' : `until ${new Date(wait.until).toISOString()}`}; ${held.length} request(s) retained`);
|
|
2468
|
+
}
|
|
2469
|
+
return true;
|
|
2470
|
+
}
|
|
2471
|
+
/**
|
|
2472
|
+
* Refuse an auxiliary call to a model the provider asked this agent to wait
|
|
2473
|
+
* on, without calling it. The refusal is a retryable rate limit carrying
|
|
2474
|
+
* the remaining wait, so a caller that paces (context-manager's compression
|
|
2475
|
+
* lane, from its corrective round #138 on) waits it out instead of spending
|
|
2476
|
+
* a call. It is marked
|
|
2477
|
+
* `providerAdmission: 'deferred'`: no call was made, so a caller can keep
|
|
2478
|
+
* the wait without counting a failed call (room-225 #47927).
|
|
2479
|
+
*/
|
|
2480
|
+
refuseHeldProviderModel(agentName, model) {
|
|
2481
|
+
// A call that names no model can't be matched to a model's wait, but the
|
|
2482
|
+
// hold that stands for unreadable records binds every model, so it binds
|
|
2483
|
+
// that call too (EVERY_MODEL: only that hold, or a release of it, says so).
|
|
2484
|
+
const wait = this.providerWaits?.active(agentName, model ?? EVERY_MODEL);
|
|
2485
|
+
if (!wait)
|
|
2486
|
+
return;
|
|
2487
|
+
const now = Date.now();
|
|
2488
|
+
throw Object.assign(new MembraneError({
|
|
2489
|
+
type: 'rate_limit',
|
|
2490
|
+
retryable: true,
|
|
2491
|
+
...(wait.until !== null ? { retryAfterMs: Math.max(0, wait.until - now) } : {}),
|
|
2492
|
+
message: providerWaitText(agentName, model ?? EVERY_MODEL, wait),
|
|
2493
|
+
rawError: undefined,
|
|
2494
|
+
}), { providerAdmission: 'deferred' });
|
|
2495
|
+
}
|
|
2496
|
+
/**
|
|
2497
|
+
* Release provider waits: one model's for an agent, or every model's. The
|
|
2498
|
+
* explicit operator override for a wait that should not be honoured (the
|
|
2499
|
+
* passing of time needs no call). Recorded, so a restart does not bring the
|
|
2500
|
+
* wait back; held primary turns are released into one fresh compile. An
|
|
2501
|
+
* acceleration hold the released wait lengthened keeps only AF's own pacing
|
|
2502
|
+
* for its failure; a host hold stays the host's.
|
|
2503
|
+
*/
|
|
2504
|
+
releaseProviderWait(agentName, model, by = 'operator') {
|
|
2505
|
+
// A release names one of this framework's agents, or the agent of a wait
|
|
2506
|
+
// the listing shows (a disposed conversation fork's wait stays recorded):
|
|
2507
|
+
// an operator can release anything the listing shows. Any other name (a
|
|
2508
|
+
// typo, or '*' and 'framework', which stand for the hold on unreadable
|
|
2509
|
+
// records in its listing and its alert) is refused before anything is
|
|
2510
|
+
// recorded: it would lift nothing and still read as a success. That hold
|
|
2511
|
+
// is lifted one agent at a time.
|
|
2512
|
+
if (!this.agents.has(agentName) && !this.providerWaits?.recorded(agentName)) {
|
|
2513
|
+
throw new Error(`Unknown agent: ${safeSlice(JSON.stringify(agentName), 0, 120)}. A release names one of this ` +
|
|
2514
|
+
"framework's agents or an agent with a recorded wait, and the hold on unreadable provider waits is lifted " +
|
|
2515
|
+
'one agent at a time');
|
|
2516
|
+
}
|
|
2517
|
+
by = operatorLabel(String(by));
|
|
2518
|
+
// '*' (EVERY_MODEL, as providerWaits lists the unreadable-records hold)
|
|
2519
|
+
// names every model, the same as omitting it, everywhere below.
|
|
2520
|
+
if (model === EVERY_MODEL)
|
|
2521
|
+
model = undefined;
|
|
2522
|
+
const released = this.providerWaits?.release(agentName, model, by) ?? [];
|
|
2523
|
+
for (const { wait, release } of released) {
|
|
2524
|
+
console.error(`[provider-wait] agent=${agentName} model=${wait.model} released by ${by} (${release})`);
|
|
2525
|
+
}
|
|
2526
|
+
// The agent's context strategy may pace its own provider calls by the
|
|
2527
|
+
// same stated wait (context-manager's compression lane, from its
|
|
2528
|
+
// corrective round #138 on; 0.14.0 has no releaseCompressionPause, so this
|
|
2529
|
+
// calls nothing there). The release is the operator's word on that wait
|
|
2530
|
+
// too, so it is offered to the lane with its scope even when no wait
|
|
2531
|
+
// binds here: the lane can hold a pause after
|
|
2532
|
+
// this framework's own wait is gone (the records read again, the wait
|
|
2533
|
+
// passed, or it was released in another process). The strategy's own
|
|
2534
|
+
// backoff stays, and the receipt still lists only waits released here.
|
|
2535
|
+
// Read inside the try: the release is already recorded, so a strategy
|
|
2536
|
+
// that throws must not keep the hold below from being released.
|
|
2537
|
+
try {
|
|
2538
|
+
const strategy = this.agents.get(agentName)?.getContextManager().getStrategy();
|
|
2539
|
+
if (strategy?.releaseCompressionPause?.(model)) {
|
|
2540
|
+
console.error(`[provider-wait] agent=${agentName} model=${model ?? '*'} compression lane's provider wait released`);
|
|
2541
|
+
}
|
|
2542
|
+
}
|
|
2543
|
+
catch (error) {
|
|
2544
|
+
console.error(`[provider-wait] agent=${agentName} could not offer the release to the compression lane: ${safeSlice(error instanceof Error ? error.message : String(error), 0, 300)}`);
|
|
2545
|
+
}
|
|
2546
|
+
const cooldown = this.providerAccelerationCooldowns.get(agentName);
|
|
2547
|
+
const covers = (held) => held !== undefined && (model === undefined || held === model);
|
|
2548
|
+
if (cooldown && covers(cooldown.waitModel)) {
|
|
2549
|
+
// A wait hold asks its wait again: released, the hold ends.
|
|
2550
|
+
this.releaseProviderAccelerationCooldown(agentName);
|
|
2551
|
+
}
|
|
2552
|
+
else if (cooldown?.statedWait && covers(cooldown.statedWait.model)) {
|
|
2553
|
+
// An acceleration hold the stated wait lengthened keeps only AF's own
|
|
2554
|
+
// pacing for that failure, and is never lengthened by a release. A host
|
|
2555
|
+
// hold carries no stated wait: its length stays the host's.
|
|
2556
|
+
cooldown.until = Math.min(cooldown.until, cooldown.statedWait.ownUntil);
|
|
2557
|
+
console.error(`[provider-wait] agent=${agentName} model=${cooldown.statedWait.model} acceleration hold keeps AF's own pacing only, ` +
|
|
2558
|
+
(cooldown.until > Date.now() ? `until ${new Date(cooldown.until).toISOString()}` : 'which has passed'));
|
|
2559
|
+
delete cooldown.statedWait;
|
|
2560
|
+
this.releaseProviderAccelerationCooldown(agentName);
|
|
2561
|
+
}
|
|
2562
|
+
return released.map(({ wait, release }) => ({
|
|
2563
|
+
model: wait.model, until: wait.until === null ? null : new Date(wait.until).toISOString(), release,
|
|
2564
|
+
}));
|
|
2565
|
+
}
|
|
2566
|
+
/** Provider waits binding now, for health and doctor tooling. */
|
|
2567
|
+
providerWaitSnapshot(agentName) {
|
|
2568
|
+
return (this.providerWaits?.list(agentName) ?? []).map((wait) => ({
|
|
2569
|
+
agent: wait.agent,
|
|
2570
|
+
model: wait.model,
|
|
2571
|
+
until: wait.until === null ? null : new Date(wait.until).toISOString(),
|
|
2572
|
+
reason: wait.reason,
|
|
2573
|
+
setAt: new Date(wait.setAt).toISOString(),
|
|
2574
|
+
}));
|
|
2575
|
+
}
|
|
1896
2576
|
recordProviderAccelerationRecovery(agent, request, stopReason) {
|
|
1897
2577
|
const recovery = this.providerAccelerationRecoveries.get(agent.name);
|
|
1898
2578
|
if (!recovery?.releasedAt)
|
|
@@ -1934,9 +2614,12 @@ export class AgentFramework {
|
|
|
1934
2614
|
if (this.maintenancePass || !this.running)
|
|
1935
2615
|
return;
|
|
1936
2616
|
const pass = this.runQueuedMaintenance().catch((error) => {
|
|
1937
|
-
|
|
2617
|
+
// The alert cuts its own copies from the whole message, so the
|
|
2618
|
+
// webhook's states the whole length.
|
|
2619
|
+
const wholeReason = error instanceof Error ? String(error.message) : failureDescription(error);
|
|
2620
|
+
const reason = boundFailureText(wholeReason);
|
|
1938
2621
|
console.error('[context-maintenance] pass failed:', reason);
|
|
1939
|
-
this.opsAlert('context-maintenance-failed', 'framework',
|
|
2622
|
+
this.opsAlert('context-maintenance-failed', 'framework', wholeReason, { data: { scope: 'pass' } });
|
|
1940
2623
|
});
|
|
1941
2624
|
this.maintenancePass = pass;
|
|
1942
2625
|
void pass.finally(() => {
|
|
@@ -2005,9 +2688,10 @@ export class AgentFramework {
|
|
|
2005
2688
|
}
|
|
2006
2689
|
}
|
|
2007
2690
|
catch (error) {
|
|
2008
|
-
|
|
2691
|
+
const wholeError = error instanceof Error ? String(error.message) : failureDescription(error);
|
|
2692
|
+
record.error = boundFailureText(wholeError);
|
|
2009
2693
|
console.error(`[context-maintenance] agent=${agent.name} failed:`, record.error);
|
|
2010
|
-
this.opsAlert('context-maintenance-failed', agent.name,
|
|
2694
|
+
this.opsAlert('context-maintenance-failed', agent.name, wholeError, {
|
|
2011
2695
|
data: {
|
|
2012
2696
|
scope: 'agent',
|
|
2013
2697
|
...(record.pendingBefore ? { pending: record.pendingBefore } : {}),
|
|
@@ -2111,19 +2795,41 @@ export class AgentFramework {
|
|
|
2111
2795
|
this.queue.push(event);
|
|
2112
2796
|
this.emitTrace({ type: 'process:received', processEvent: event });
|
|
2113
2797
|
}
|
|
2798
|
+
/**
|
|
2799
|
+
* A module's event. stop() closes the queue before it stops modules, so a
|
|
2800
|
+
* module's event can arrive after that: from a listener that fires while
|
|
2801
|
+
* stop() runs, or from the module's own stop(). Nothing will process it,
|
|
2802
|
+
* and a throw there can go unhandled or fail stop(), so it's dropped with a
|
|
2803
|
+
* log line naming the module. Otherwise it's queued as pushEvent() queues
|
|
2804
|
+
* it, or, through the deprecated queue handle, without the trace.
|
|
2805
|
+
*/
|
|
2806
|
+
pushModuleEvent(event, moduleName, traced) {
|
|
2807
|
+
if (this.queue.isClosed) {
|
|
2808
|
+
// A tool result takes pushEvent()'s own drop, whose line names the
|
|
2809
|
+
// call, the tool and its outcome.
|
|
2810
|
+
if (event.type === 'tool-result') {
|
|
2811
|
+
this.pushEvent(event);
|
|
2812
|
+
return;
|
|
2813
|
+
}
|
|
2814
|
+
console.error(`[framework] a '${event.type}' event from module ${moduleName ?? '(unnamed)'} arrived after stop() began — dropped`);
|
|
2815
|
+
return;
|
|
2816
|
+
}
|
|
2817
|
+
if (traced)
|
|
2818
|
+
this.pushEvent(event);
|
|
2819
|
+
else
|
|
2820
|
+
this.queue.push(event);
|
|
2821
|
+
}
|
|
2114
2822
|
/**
|
|
2115
2823
|
* Add a trace event listener for observability.
|
|
2116
2824
|
*/
|
|
2117
2825
|
onTrace(listener) {
|
|
2118
|
-
|
|
2826
|
+
const registration = { listener, active: true };
|
|
2827
|
+
this.traceListeners.push(registration);
|
|
2119
2828
|
// Return an unsubscribe so callers with a bounded lifetime (e.g. modules
|
|
2120
2829
|
// that get torn down and recreated on session switch) don't leak
|
|
2121
2830
|
// listeners. Existing callers that ignore the return value are unaffected.
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
if (idx >= 0)
|
|
2125
|
-
this.traceListeners.splice(idx, 1);
|
|
2126
|
-
};
|
|
2831
|
+
// It removes this subscription, whatever else the same function holds.
|
|
2832
|
+
return () => this.removeTraceRegistration(registration);
|
|
2127
2833
|
}
|
|
2128
2834
|
/**
|
|
2129
2835
|
* Public accessor for the MCPL channel registry.
|
|
@@ -2135,13 +2841,19 @@ export class AgentFramework {
|
|
|
2135
2841
|
return this.channelRegistry;
|
|
2136
2842
|
}
|
|
2137
2843
|
/**
|
|
2138
|
-
* Remove a trace event listener
|
|
2844
|
+
* Remove a trace event listener: its first subscription, if it holds more
|
|
2845
|
+
* than one.
|
|
2139
2846
|
*/
|
|
2140
2847
|
offTrace(listener) {
|
|
2141
|
-
const
|
|
2142
|
-
if (
|
|
2848
|
+
const registration = this.traceListeners.find((r) => r.listener === listener);
|
|
2849
|
+
if (registration)
|
|
2850
|
+
this.removeTraceRegistration(registration);
|
|
2851
|
+
}
|
|
2852
|
+
removeTraceRegistration(registration) {
|
|
2853
|
+
registration.active = false;
|
|
2854
|
+
const index = this.traceListeners.indexOf(registration);
|
|
2855
|
+
if (index >= 0)
|
|
2143
2856
|
this.traceListeners.splice(index, 1);
|
|
2144
|
-
}
|
|
2145
2857
|
}
|
|
2146
2858
|
/**
|
|
2147
2859
|
* Add a module at runtime.
|
|
@@ -2280,16 +2992,29 @@ export class AgentFramework {
|
|
|
2280
2992
|
async autoFetchReferences(content, serverId, eager) {
|
|
2281
2993
|
if (!content)
|
|
2282
2994
|
return;
|
|
2283
|
-
|
|
2284
|
-
|
|
2995
|
+
// Register every block first, then judge each uri once, by its latest
|
|
2996
|
+
// record. The registry keys records by uri and evicts the one registered
|
|
2997
|
+
// least recently, so a uri this result mentions keeps its record through
|
|
2998
|
+
// the result's later blocks, short of more distinct references in one
|
|
2999
|
+
// result than the registry holds; past that, a uri can get a second
|
|
3000
|
+
// record here, hence the key by uri. A uri that any of this result's
|
|
3001
|
+
// blocks calls expired takes no slot, whichever block registered last. A
|
|
3002
|
+
// later result's fresh expiry still supersedes, through register().
|
|
3003
|
+
const byUri = new Map();
|
|
2285
3004
|
for (const block of content) {
|
|
2286
3005
|
const c = classifyBlock(block);
|
|
2287
3006
|
if (c.kind !== 'reference')
|
|
2288
3007
|
continue;
|
|
2289
3008
|
const record = referenceRegistry.register(c.testimony, serverId);
|
|
2290
|
-
|
|
2291
|
-
|
|
3009
|
+
const expiredHere = byUri.get(c.testimony.uri)?.expiredHere === true || isReferenceExpired(c.testimony);
|
|
3010
|
+
byUri.set(c.testimony.uri, { record, expiredHere });
|
|
2292
3011
|
}
|
|
3012
|
+
if (!eager)
|
|
3013
|
+
return;
|
|
3014
|
+
const fetcher = this.getReferenceFetcher();
|
|
3015
|
+
const eligible = [...byUri.values()]
|
|
3016
|
+
.filter(({ record, expiredHere }) => !expiredHere && fetcher.isEagerEligible(record))
|
|
3017
|
+
.map(({ record }) => record);
|
|
2293
3018
|
// Parallel with a count cap: N small fetches cost one round-trip of
|
|
2294
3019
|
// wall-clock, and anything past the cap stays lazy (stub says so).
|
|
2295
3020
|
const MAX_EAGER_PER_RESULT = 8;
|
|
@@ -3885,7 +4610,8 @@ export class AgentFramework {
|
|
|
3885
4610
|
return;
|
|
3886
4611
|
run.lastActivity = Date.now();
|
|
3887
4612
|
if (outcome.stopReason === 'exhausted') {
|
|
3888
|
-
|
|
4613
|
+
// The caller may log or relay the message, which can echo the whole request.
|
|
4614
|
+
run.settle.reject(new Error(boundFailureText(outcome.error ?? 'Unknown error')));
|
|
3889
4615
|
return;
|
|
3890
4616
|
}
|
|
3891
4617
|
run.settle.resolve({
|
|
@@ -4415,7 +5141,8 @@ export class AgentFramework {
|
|
|
4415
5141
|
if (parts[0] !== 'discord')
|
|
4416
5142
|
return;
|
|
4417
5143
|
const channelId = parts[parts.length - 1];
|
|
4418
|
-
|
|
5144
|
+
// Own names only: an inherited one such as 'constructor' takes the fallback.
|
|
5145
|
+
const emoji = Object.hasOwn(REFUSAL_REACTIONS, category) ? REFUSAL_REACTIONS[category] : REFUSAL_REACTION_FALLBACK;
|
|
4419
5146
|
// Resolve the MCPL server that owns the locus channel and call
|
|
4420
5147
|
// tools/call directly on its connection (bare tool name — no prefix
|
|
4421
5148
|
// games), bypassing the agent event queue so no synthetic tool-result
|
|
@@ -4528,9 +5255,10 @@ export class AgentFramework {
|
|
|
4528
5255
|
? `the model refused on them (content filter: ${category}). Their content is ` +
|
|
4529
5256
|
`not reproduced here (so this note can't re-trigger the filter); the ` +
|
|
4530
5257
|
`originals remain in the raw record`
|
|
4531
|
-
: `the model API kept
|
|
4532
|
-
`
|
|
4533
|
-
`
|
|
5258
|
+
: `the model API kept refusing the requests that carried them (${category}), ` +
|
|
5259
|
+
`so that the conversation could go on. Their content is not reproduced ` +
|
|
5260
|
+
`here (so this note can't trigger the same refusal); the originals remain ` +
|
|
5261
|
+
`in the raw record`;
|
|
4534
5262
|
const text = `[refusal-rewind] The ${count} most recent turn(s) were set aside because ` +
|
|
4535
5263
|
`${why}. You are clear to continue — if you ` +
|
|
4536
5264
|
`were mid-task, take a different approach; otherwise carry on with whatever ` +
|
|
@@ -4675,6 +5403,10 @@ export class AgentFramework {
|
|
|
4675
5403
|
*
|
|
4676
5404
|
* host-status — report the quiesce state (drained / parked wakes /
|
|
4677
5405
|
* running background scripts).
|
|
5406
|
+
*
|
|
5407
|
+
* release-provider-wait — end an agent's provider wait (a provider's
|
|
5408
|
+
* stated retry-after, kept across restarts) for one `model` or all, as an
|
|
5409
|
+
* explicit, recorded operator override.
|
|
4678
5410
|
*/
|
|
4679
5411
|
async handleHostCommand(serverId, params) {
|
|
4680
5412
|
if (params.command !== 'undo' &&
|
|
@@ -4684,7 +5416,8 @@ export class AgentFramework {
|
|
|
4684
5416
|
params.command !== 'quiesce' &&
|
|
4685
5417
|
params.command !== 'resume' &&
|
|
4686
5418
|
params.command !== 'maintain' &&
|
|
4687
|
-
params.command !== 'host-status'
|
|
5419
|
+
params.command !== 'host-status' &&
|
|
5420
|
+
params.command !== 'release-provider-wait') {
|
|
4688
5421
|
return { ok: false, error: `Unknown host command: ${String(params.command)}` };
|
|
4689
5422
|
}
|
|
4690
5423
|
// Host-scoped verbs (issue #122) — no agent resolution.
|
|
@@ -4720,6 +5453,31 @@ export class AgentFramework {
|
|
|
4720
5453
|
if (params.command === 'host-status') {
|
|
4721
5454
|
return { ok: true, hostMode: this.getHostModeStatus() };
|
|
4722
5455
|
}
|
|
5456
|
+
// A release names its agent, as host.releaseProviderWait does: the first
|
|
5457
|
+
// agent by default could lift the wrong agent's waits. The name may be an
|
|
5458
|
+
// agent the framework no longer has whose wait the listing still shows
|
|
5459
|
+
// (a disposed conversation fork); releaseProviderWait decides that.
|
|
5460
|
+
if (params.command === 'release-provider-wait') {
|
|
5461
|
+
if (typeof params.agentName !== 'string' || params.agentName === '') {
|
|
5462
|
+
return { ok: false, error: 'release-provider-wait: agentName must name the agent whose waits to release' };
|
|
5463
|
+
}
|
|
5464
|
+
// Only an omitted model, or '*', releases every model: a malformed one
|
|
5465
|
+
// is refused here, before anything changes, never widened.
|
|
5466
|
+
if (!isReleaseModel(params.model)) {
|
|
5467
|
+
return { ok: false, error: "release-provider-wait: model must be a model name, or '*' (or omitted) for every model" };
|
|
5468
|
+
}
|
|
5469
|
+
const by = operatorLabel(String(requester));
|
|
5470
|
+
let released;
|
|
5471
|
+
try {
|
|
5472
|
+
released = this.releaseProviderWait(params.agentName, params.model, by);
|
|
5473
|
+
}
|
|
5474
|
+
catch (error) {
|
|
5475
|
+
return { ok: false, error: error instanceof Error ? error.message : String(error) };
|
|
5476
|
+
}
|
|
5477
|
+
console.error(`[host-command] release-provider-wait agent=${params.agentName} model=${params.model ?? '*'} by=${by} ` +
|
|
5478
|
+
`(server=${serverId}): ${released.length} wait(s) lifted`);
|
|
5479
|
+
return { ok: true, released };
|
|
5480
|
+
}
|
|
4723
5481
|
const agentName = params.agentName ?? [...this.agents.keys()][0];
|
|
4724
5482
|
if (!agentName || !this.agents.has(agentName)) {
|
|
4725
5483
|
return { ok: false, error: `Unknown agent: ${String(agentName)}` };
|
|
@@ -4884,6 +5642,7 @@ export class AgentFramework {
|
|
|
4884
5642
|
ok: true,
|
|
4885
5643
|
messagesRemoved: r.messagesRemoved,
|
|
4886
5644
|
removedRefs: r.removedRefs,
|
|
5645
|
+
markers: r.markers,
|
|
4887
5646
|
lastVisible: r.lastVisible,
|
|
4888
5647
|
};
|
|
4889
5648
|
}
|
|
@@ -4918,7 +5677,9 @@ export class AgentFramework {
|
|
|
4918
5677
|
if (undone === 0) {
|
|
4919
5678
|
return { ok: true, undone: 0, requested, lastVisible: null };
|
|
4920
5679
|
}
|
|
4921
|
-
|
|
5680
|
+
// The branch has moved: delivery of the marks it changes starts behind
|
|
5681
|
+
// the data-plane gate, and the command doesn't wait for Discord.
|
|
5682
|
+
this.startDiscordAwarenessDelivery('turn undo', agentName);
|
|
4922
5683
|
return { ok: true, undone, requested, lastVisible: await this.lastVisiblePreview(agentName) };
|
|
4923
5684
|
}
|
|
4924
5685
|
/**
|
|
@@ -5434,6 +6195,12 @@ export class AgentFramework {
|
|
|
5434
6195
|
* Discord messages that left the live context get awareness markers via the
|
|
5435
6196
|
* durable outbox, exactly as message-granular `undo` does.
|
|
5436
6197
|
*
|
|
6198
|
+
* Returns once the switch has landed: the body is applied, with a
|
|
6199
|
+
* marker-scheduling receipt (`markers`). It does not wait for Discord to
|
|
6200
|
+
* accept the reactions: delivery is started, behind the MCPL data-plane
|
|
6201
|
+
* gate, and continues without the caller or the store reservation waiting
|
|
6202
|
+
* on it; its per-message outcomes stay in the outbox ledger.
|
|
6203
|
+
*
|
|
5437
6204
|
* Throws `OperatorActionError` (`agent-busy`, `unknown-message`, …) — the
|
|
5438
6205
|
* agent must be idle; nothing is queued.
|
|
5439
6206
|
*/
|
|
@@ -5473,9 +6240,12 @@ export class AgentFramework {
|
|
|
5473
6240
|
throw new OperatorActionError('invalid', 'Rollback branch name must differ from the active branch');
|
|
5474
6241
|
}
|
|
5475
6242
|
// Gate + reserve the whole store (see reserveStoreForSurgery); held
|
|
5476
|
-
// until the switch has landed
|
|
6243
|
+
// until the switch has landed and its marker scheduling has been
|
|
6244
|
+
// recorded or reported, or the switch has been rolled back. Remote
|
|
6245
|
+
// delivery is not waited on.
|
|
5477
6246
|
const release = this.reserveStoreForSurgery('roll back', agentName);
|
|
5478
6247
|
let branchName;
|
|
6248
|
+
let markers;
|
|
5479
6249
|
try {
|
|
5480
6250
|
// Prepare the external side effect before switching Chronicle. If the
|
|
5481
6251
|
// process dies after the switch but before activate(), startup
|
|
@@ -5503,9 +6273,14 @@ export class AgentFramework {
|
|
|
5503
6273
|
const restored = await this.restoreSourceBranch(cm, sourceBranch, targetBranch, agentName);
|
|
5504
6274
|
throw new OperatorActionError('failed', `Rollback failed; ${restored}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
5505
6275
|
}
|
|
5506
|
-
|
|
5507
|
-
|
|
5508
|
-
|
|
6276
|
+
// The body change has landed. Nothing after this point turns the
|
|
6277
|
+
// rollback into a failure: marks are an outbox outcome, reported
|
|
6278
|
+
// apart from the body (`markers`).
|
|
6279
|
+
markers = this.activateSurgeryMarkers(markerBatch, 'rollback', agentName);
|
|
6280
|
+
// Installs the MCPL data-plane gate synchronously, before the store is
|
|
6281
|
+
// released, and starts delivery without awaiting it (the first
|
|
6282
|
+
// reaction request may already be on the wire when this returns).
|
|
6283
|
+
this.startDiscordAwarenessDelivery('rollback', agentName);
|
|
5509
6284
|
this.materializeConfigMountAfterBranchSwitch();
|
|
5510
6285
|
}
|
|
5511
6286
|
finally {
|
|
@@ -5513,7 +6288,8 @@ export class AgentFramework {
|
|
|
5513
6288
|
}
|
|
5514
6289
|
console.error(`[operator] rollback agent=${agentName} to=${tailMessageId}` +
|
|
5515
6290
|
`${tailMessageId !== opts.messageId ? ` (requested ${opts.messageId}, snapped to end of body group)` : ''}` +
|
|
5516
|
-
` removed=${messagesRemoved} branch=${branchName}
|
|
6291
|
+
` removed=${messagesRemoved} branch=${branchName} marks=${describeMarkers(markers)}` +
|
|
6292
|
+
` by=${describeRequester(opts.requester)}`);
|
|
5517
6293
|
this.recordOperatorAction({
|
|
5518
6294
|
...logBase,
|
|
5519
6295
|
result: {
|
|
@@ -5522,6 +6298,7 @@ export class AgentFramework {
|
|
|
5522
6298
|
tailMessageId,
|
|
5523
6299
|
messagesRemoved,
|
|
5524
6300
|
discordRefs: removedRefs.length,
|
|
6301
|
+
markers,
|
|
5525
6302
|
},
|
|
5526
6303
|
});
|
|
5527
6304
|
return {
|
|
@@ -5531,6 +6308,7 @@ export class AgentFramework {
|
|
|
5531
6308
|
messagesRemoved,
|
|
5532
6309
|
tailMessageId,
|
|
5533
6310
|
removedRefs,
|
|
6311
|
+
markers,
|
|
5534
6312
|
lastVisible: await this.lastVisiblePreview(agentName),
|
|
5535
6313
|
};
|
|
5536
6314
|
}
|
|
@@ -5546,6 +6324,9 @@ export class AgentFramework {
|
|
|
5546
6324
|
* removed together. Discord originals get awareness markers via the outbox;
|
|
5547
6325
|
* the batch is `explicit`-activated only after every removal succeeded, and
|
|
5548
6326
|
* a crash mid-way is finished at next boot (resumePreparedDiscordSuppressions).
|
|
6327
|
+
* Like rollbackToMessage, it returns once the body change has landed, with
|
|
6328
|
+
* a marker-scheduling receipt (`markers`); delivery continues without the
|
|
6329
|
+
* caller waiting on it.
|
|
5549
6330
|
*
|
|
5550
6331
|
* Not retroactive over derived state: a message already folded into an
|
|
5551
6332
|
* autobiographical summary stays in that summary — roll back to before it
|
|
@@ -5596,9 +6377,12 @@ export class AgentFramework {
|
|
|
5596
6377
|
throw new OperatorActionError('invalid', 'Suppression branch name must differ from the active branch');
|
|
5597
6378
|
}
|
|
5598
6379
|
// Gate + reserve the whole store (see reserveStoreForSurgery); held
|
|
5599
|
-
// until the fork is fully redacted
|
|
6380
|
+
// until the fork is fully redacted and its marker scheduling has been
|
|
6381
|
+
// recorded or reported, or the source is restored. Remote delivery is
|
|
6382
|
+
// not waited on.
|
|
5600
6383
|
const release = this.reserveStoreForSurgery('suppress', agentName);
|
|
5601
6384
|
let createdBranch;
|
|
6385
|
+
let markers;
|
|
5602
6386
|
try {
|
|
5603
6387
|
const markerBatch = this.discordAwarenessOutbox?.prepare({
|
|
5604
6388
|
agentName,
|
|
@@ -5638,8 +6422,6 @@ export class AgentFramework {
|
|
|
5638
6422
|
else
|
|
5639
6423
|
cm.removeMessage(iv.fromId);
|
|
5640
6424
|
}
|
|
5641
|
-
if (markerBatch)
|
|
5642
|
-
this.discordAwarenessOutbox.activate(markerBatch.id);
|
|
5643
6425
|
}
|
|
5644
6426
|
catch (error) {
|
|
5645
6427
|
// A partially suppressed branch is not safe to serve from. Keep it
|
|
@@ -5649,14 +6431,26 @@ export class AgentFramework {
|
|
|
5649
6431
|
const restored = await this.restoreSourceBranch(cm, sourceBranch, createdBranch, agentName);
|
|
5650
6432
|
throw new OperatorActionError('failed', `Suppression failed on ${createdBranch}; ${restored}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
5651
6433
|
}
|
|
5652
|
-
|
|
6434
|
+
// Every redaction committed: the body change has landed and the fork
|
|
6435
|
+
// is safe to serve from. Activating the marker batch is bookkeeping
|
|
6436
|
+
// about a world-side effect; if it fails, the suppression still
|
|
6437
|
+
// stands, and the receipt says what became of the marks: not
|
|
6438
|
+
// scheduled once the batch is retired, or unresolved if it can't be.
|
|
6439
|
+
// (The batch's resume journal has nothing left to do, since every
|
|
6440
|
+
// interval has committed, so retiring the batch gives up only the
|
|
6441
|
+
// marks.)
|
|
6442
|
+
markers = this.activateSurgeryMarkers(markerBatch, 'suppress', agentName);
|
|
6443
|
+
// Installs the MCPL data-plane gate synchronously, before the store is
|
|
6444
|
+
// released, and starts delivery without awaiting it (the first
|
|
6445
|
+
// reaction request may already be on the wire when this returns).
|
|
6446
|
+
this.startDiscordAwarenessDelivery('suppress', agentName);
|
|
5653
6447
|
this.materializeConfigMountAfterBranchSwitch();
|
|
5654
6448
|
}
|
|
5655
6449
|
finally {
|
|
5656
6450
|
release();
|
|
5657
6451
|
}
|
|
5658
6452
|
console.error(`[operator] suppress agent=${agentName} removed=${messagesRemoved} branch=${createdBranch} ` +
|
|
5659
|
-
`by=${describeRequester(opts.requester)}`);
|
|
6453
|
+
`marks=${describeMarkers(markers)} by=${describeRequester(opts.requester)}`);
|
|
5660
6454
|
const loggedIds = capIds(removedIds);
|
|
5661
6455
|
this.recordOperatorAction({
|
|
5662
6456
|
...logBase,
|
|
@@ -5667,6 +6461,7 @@ export class AgentFramework {
|
|
|
5667
6461
|
removedIds: loggedIds.ids,
|
|
5668
6462
|
...(loggedIds.truncated ? { removedIdsTruncated: true } : {}),
|
|
5669
6463
|
discordRefs: removedRefs.length,
|
|
6464
|
+
markers,
|
|
5670
6465
|
},
|
|
5671
6466
|
});
|
|
5672
6467
|
return {
|
|
@@ -5676,6 +6471,7 @@ export class AgentFramework {
|
|
|
5676
6471
|
messagesRemoved,
|
|
5677
6472
|
removedIds,
|
|
5678
6473
|
removedRefs,
|
|
6474
|
+
markers,
|
|
5679
6475
|
lastVisible: await this.lastVisiblePreview(agentName),
|
|
5680
6476
|
};
|
|
5681
6477
|
}
|
|
@@ -5684,6 +6480,97 @@ export class AgentFramework {
|
|
|
5684
6480
|
throw error;
|
|
5685
6481
|
}
|
|
5686
6482
|
}
|
|
6483
|
+
/**
|
|
6484
|
+
* Record a surgery's prepared marker batch as active once its body change
|
|
6485
|
+
* has landed, and say what happened. Never throws: the body is already
|
|
6486
|
+
* applied, so a ledger failure here is reported in the receipt rather than
|
|
6487
|
+
* as a failed surgery an operator might retry. A batch that could not be
|
|
6488
|
+
* activated is retired, so no later reconciliation can deliver it behind a
|
|
6489
|
+
* receipt saying it was not scheduled (`not-scheduled`). If retiring fails
|
|
6490
|
+
* too, the receipt says only that the outcome is `unresolved`: the batch is
|
|
6491
|
+
* retained, and it may still be delivered. A rollback's batch
|
|
6492
|
+
* (activated by target branch) may be promoted by the very next
|
|
6493
|
+
* reconciliation, the one the delivery it starts runs; a suppression's
|
|
6494
|
+
* (activated explicitly) waits for the next start.
|
|
6495
|
+
*
|
|
6496
|
+
* A batch with no refs (a suppression journal with nothing addressable) is
|
|
6497
|
+
* still activated, which closes its resume journal. It schedules no marks,
|
|
6498
|
+
* so it reports `none` whatever the ledger does. If it can be neither
|
|
6499
|
+
* activated nor retired, its journal is left prepared and closes at the
|
|
6500
|
+
* next start: every interval it names has already committed, so the resume
|
|
6501
|
+
* skips them all and activates it.
|
|
6502
|
+
*/
|
|
6503
|
+
activateSurgeryMarkers(batch, verb, agentName) {
|
|
6504
|
+
if (!batch)
|
|
6505
|
+
return { status: 'none', queued: 0 };
|
|
6506
|
+
try {
|
|
6507
|
+
this.discordAwarenessOutbox.activate(batch.id);
|
|
6508
|
+
}
|
|
6509
|
+
catch (error) {
|
|
6510
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
6511
|
+
let retired = false;
|
|
6512
|
+
let retireDetail = '';
|
|
6513
|
+
try {
|
|
6514
|
+
retired = this.discordAwarenessOutbox.discard(batch.id);
|
|
6515
|
+
}
|
|
6516
|
+
catch (discardError) {
|
|
6517
|
+
retireDetail = discardError instanceof Error ? discardError.message : String(discardError);
|
|
6518
|
+
}
|
|
6519
|
+
console.error(`[discord-awareness] ${verb} agent=${agentName}: body applied, but marker batch ${batch.id} ` +
|
|
6520
|
+
`could not be activated (${detail}); ` +
|
|
6521
|
+
(retired ? 'batch retired' : `batch retained${retireDetail ? ` (retire failed: ${retireDetail})` : ''}`));
|
|
6522
|
+
if (batch.refs.length === 0)
|
|
6523
|
+
return { status: 'none', queued: 0 };
|
|
6524
|
+
this.opsAlert(retired ? 'discord-awareness-not-scheduled' : 'discord-awareness-unresolved', agentName, retired
|
|
6525
|
+
? `${verb} applied, but its ${batch.refs.length} awareness mark(s) were not scheduled ` +
|
|
6526
|
+
`(batch ${batch.id} retired): ${detail}`
|
|
6527
|
+
: `${verb} applied, but awareness batch ${batch.id} (${batch.refs.length} mark(s)) could be ` +
|
|
6528
|
+
`neither activated nor retired; it may still be delivered: ${detail}`, { data: { batchId: batch.id, retired } });
|
|
6529
|
+
return retired
|
|
6530
|
+
? { status: 'not-scheduled', queued: 0, error: detail }
|
|
6531
|
+
: { status: 'unresolved', queued: 0, batchId: batch.id, error: detail };
|
|
6532
|
+
}
|
|
6533
|
+
if (batch.refs.length === 0)
|
|
6534
|
+
return { status: 'none', queued: 0 };
|
|
6535
|
+
return { status: 'queued', queued: batch.refs.length, batchId: batch.id };
|
|
6536
|
+
}
|
|
6537
|
+
/**
|
|
6538
|
+
* Start awareness delivery after a branch move (a surgery, a switch, an
|
|
6539
|
+
* undo or redo) without the caller awaiting it. No operator response waits
|
|
6540
|
+
* for Discord to accept marks: the MCPL data-plane gate alone holds channel
|
|
6541
|
+
* traffic that arrives after it until they are attempted.
|
|
6542
|
+
* `syncDiscordAwarenessMarkers` installs that gate synchronously, before
|
|
6543
|
+
* its first await, so calling this while a surgery still holds the store
|
|
6544
|
+
* puts the gate up before the store is released. With no drain running, the
|
|
6545
|
+
* drain starts in the same synchronous run (the first reaction request can
|
|
6546
|
+
* be written before this returns); with one running, this generation's pass
|
|
6547
|
+
* follows it (drainForGeneration). Either way delivery continues
|
|
6548
|
+
* independently: the caller's response and any store reservation stop
|
|
6549
|
+
* waiting on remote reactions. A turn may run during delivery when its wake
|
|
6550
|
+
* carries no channel traffic (a scheduler timer, a provider admission), and
|
|
6551
|
+
* also when its channel traffic was admitted before the gate went up
|
|
6552
|
+
* (queued when a surgery began, or arriving during its switch or fork): the
|
|
6553
|
+
* surgery's store reservation, which used to hold such a turn through the
|
|
6554
|
+
* drain, now ends once the gate is up. A failure is accounted by the gate
|
|
6555
|
+
* itself: the current generation's failure recycles the connections, and a
|
|
6556
|
+
* superseded one leaves that to the newer generation, whose own pass meets
|
|
6557
|
+
* the same ledger. Here it is logged and raised as an ops alert, and the
|
|
6558
|
+
* next sync, reconnect or start delivers what is still pending.
|
|
6559
|
+
*
|
|
6560
|
+
* @param after What moved the branch, for the log and the alert
|
|
6561
|
+
* ("rollback", "branch switch", "undo" …).
|
|
6562
|
+
* @param agentName The agent the move was for, or "framework" for a
|
|
6563
|
+
* branch switch that names none.
|
|
6564
|
+
*/
|
|
6565
|
+
startDiscordAwarenessDelivery(after, agentName = 'framework') {
|
|
6566
|
+
if (!this.discordAwarenessOutbox)
|
|
6567
|
+
return;
|
|
6568
|
+
this.syncDiscordAwarenessMarkers().catch((error) => {
|
|
6569
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
6570
|
+
console.error(`[discord-awareness] delivery after ${after} agent=${agentName} failed: ${detail}`);
|
|
6571
|
+
this.opsAlert('discord-awareness-delivery', agentName, `Awareness mark delivery after ${after} failed: ${detail}`);
|
|
6572
|
+
});
|
|
6573
|
+
}
|
|
5687
6574
|
/** Re-materialize config files from the (new) active branch — fire and
|
|
5688
6575
|
* forget; the gate picks changes up via mtime. Mirrors undoLastTurn. */
|
|
5689
6576
|
materializeConfigMountAfterBranchSwitch() {
|
|
@@ -6068,7 +6955,7 @@ export class AgentFramework {
|
|
|
6068
6955
|
// divergence breaks the compile prefix).
|
|
6069
6956
|
const { blocks: toolResultContent, spilled } = await this.buildStoredToolResultContent(agent.name, currentState.toolResults, maxChars);
|
|
6070
6957
|
const membraneResults = currentState.toolResults.map(tc => this.toMembraneToolResult(tc.id, tc.result, maxChars, spilled.get(tc.id)));
|
|
6071
|
-
agent.toolResultGuard.storeResults(toolResultContent, membraneResults, currentState.toolResults, turnRowMetadata);
|
|
6958
|
+
agent.toolResultGuard.storeResults(toolResultContent, membraneResults, currentState.toolResults, turnRowMetadata, maxChars);
|
|
6072
6959
|
// Flush any messages that were deferred while this turn was in
|
|
6073
6960
|
// flight. Route to the PRIMARY agent — deferred messages are
|
|
6074
6961
|
// framework-level (module messages, push events, subagent return
|
|
@@ -6675,10 +7562,10 @@ export class AgentFramework {
|
|
|
6675
7562
|
isolate: true,
|
|
6676
7563
|
// Strategy instances are stateful — never share the template's.
|
|
6677
7564
|
strategy: router.strategyFactory?.() ?? new PassthroughStrategy(),
|
|
6678
|
-
//
|
|
6679
|
-
//
|
|
6680
|
-
//
|
|
6681
|
-
membrane: this.
|
|
7565
|
+
// Provider cooldown ownership belongs to the persistent resident only;
|
|
7566
|
+
// generation-unique forks must not leak gates. A fork's own recorded
|
|
7567
|
+
// provider waits still bind its compression calls.
|
|
7568
|
+
membrane: this.conversationAuxiliaryMembraneFor(name),
|
|
6682
7569
|
debugLogContext: !!process.env.DEBUG_CONTEXT,
|
|
6683
7570
|
});
|
|
6684
7571
|
// Seed with the template's compiled context, renaming the template
|
|
@@ -6745,6 +7632,18 @@ export class AgentFramework {
|
|
|
6745
7632
|
this.evictTurnCheckpoints(agentName);
|
|
6746
7633
|
if (agent)
|
|
6747
7634
|
this.logicalTurnToolCalls.delete(agent);
|
|
7635
|
+
// A provider-wait hold (nonOwningProviderWait) goes with the agent: left,
|
|
7636
|
+
// its timer would requeue held wakes for an agent that is gone, and a wait
|
|
7637
|
+
// held until released would re-arm for the name indefinitely. The recorded
|
|
7638
|
+
// wait stays, and binds the name if its channel binds again.
|
|
7639
|
+
const cooldown = this.providerAccelerationCooldowns.get(agentName);
|
|
7640
|
+
if (cooldown) {
|
|
7641
|
+
clearTimeout(cooldown.timer);
|
|
7642
|
+
this.providerAccelerationCooldowns.delete(agentName);
|
|
7643
|
+
console.error(`[provider-wait] agent=${agentName} disposed while held` +
|
|
7644
|
+
`${cooldown.waitModel !== undefined ? ` on ${cooldown.waitModel}` : ''}: ` +
|
|
7645
|
+
`${cooldown.heldRequests.length} held request(s) dropped with it (the recorded wait stays)`);
|
|
7646
|
+
}
|
|
6748
7647
|
this.emitTrace({
|
|
6749
7648
|
type: 'mcpl:conversation-disposed',
|
|
6750
7649
|
agentName,
|
|
@@ -7587,7 +8486,10 @@ export class AgentFramework {
|
|
|
7587
8486
|
}
|
|
7588
8487
|
const providerCooldown = this.providerAccelerationCooldowns?.get(agentName);
|
|
7589
8488
|
if (providerCooldown) {
|
|
7590
|
-
|
|
8489
|
+
// A provider wait binds its own model; with a different primary model
|
|
8490
|
+
// selected now, the turn proceeds (the wait stays recorded).
|
|
8491
|
+
const otherModel = providerCooldown.waitModel !== undefined && providerCooldown.waitModel !== agent.model;
|
|
8492
|
+
if (!otherModel && (now < providerCooldown.until || this.extendHostProviderHold(agentName, providerCooldown))) {
|
|
7591
8493
|
for (const req of requests) {
|
|
7592
8494
|
if (!providerCooldown.heldRequests.some((r) => this.sameInferenceRequest(r, req))) {
|
|
7593
8495
|
providerCooldown.heldRequests.push(req);
|
|
@@ -7608,6 +8510,13 @@ export class AgentFramework {
|
|
|
7608
8510
|
recovery.releasedAt = now;
|
|
7609
8511
|
requests = [...providerCooldown.heldRequests, ...requests];
|
|
7610
8512
|
}
|
|
8513
|
+
// A provider's stated wait on this agent's primary model, including one
|
|
8514
|
+
// recorded before a restart, holds the turn until it passes. An
|
|
8515
|
+
// ephemeral run records nothing, its name being single-use, so no wait,
|
|
8516
|
+
// known or unreadable, binds it (as at the last boundary).
|
|
8517
|
+
const providerWait = this.providerWaits?.active(agentName, agent.model);
|
|
8518
|
+
if (providerWait && !this.ephemeralRuns.has(agentName) && this.holdForProviderWait(agent, providerWait, requests))
|
|
8519
|
+
continue;
|
|
7611
8520
|
// Skip if agent is busy (inferring, streaming, or waiting for tools) —
|
|
7612
8521
|
// or if a turn is alive at all (activeTurnTokens): the state machine
|
|
7613
8522
|
// goes idle at endTurn's reset while driveStream teardown is still
|
|
@@ -8325,6 +9234,13 @@ export class AgentFramework {
|
|
|
8325
9234
|
}
|
|
8326
9235
|
async startAgentStream(agent, trigger, attempt = 0, providerGateAlreadyHeld = false) {
|
|
8327
9236
|
const ownsProviderGate = !this.ephemeralRuns.has(agent.name) && !this.conversationAgentHomes.has(agent.name);
|
|
9237
|
+
// A stopping framework starts no turn, whoever owns provider admission
|
|
9238
|
+
// (conversation agents and ephemeral runs do not take the gate).
|
|
9239
|
+
if (this.providerAdmissionClosed) {
|
|
9240
|
+
if (ownsProviderGate && providerGateAlreadyHeld)
|
|
9241
|
+
this.releasePrimaryProviderGate(agent.name);
|
|
9242
|
+
return;
|
|
9243
|
+
}
|
|
8328
9244
|
if (ownsProviderGate && !providerGateAlreadyHeld) {
|
|
8329
9245
|
this.acquirePrimaryProviderGate(agent.name);
|
|
8330
9246
|
const gate = this.providerGate(agent.name);
|
|
@@ -8371,7 +9287,7 @@ export class AgentFramework {
|
|
|
8371
9287
|
await this.startAgentStream(agent, trigger, attempt, true);
|
|
8372
9288
|
}).catch((error) => {
|
|
8373
9289
|
this.releasePrimaryProviderGate(agent.name);
|
|
8374
|
-
console.error(`[provider-cooldown] failed to resume primary for ${agent.name}
|
|
9290
|
+
console.error(`[provider-cooldown] failed to resume primary for ${agent.name}: ${describeFailureWithFrames(error)}`);
|
|
8375
9291
|
});
|
|
8376
9292
|
return;
|
|
8377
9293
|
}
|
|
@@ -8380,6 +9296,36 @@ export class AgentFramework {
|
|
|
8380
9296
|
return;
|
|
8381
9297
|
}
|
|
8382
9298
|
}
|
|
9299
|
+
// The last admission boundary before this primary call. With the gate
|
|
9300
|
+
// held and no auxiliary call in flight, no new provider wait can be
|
|
9301
|
+
// recorded for this agent before the call (auxiliary admission waits on
|
|
9302
|
+
// the gate). But one may have been recorded since the scheduler's check:
|
|
9303
|
+
// an auxiliary call this wake parked behind can fail with a stated wait
|
|
9304
|
+
// on the primary model. That wait binds this wake too: hold it, don't call.
|
|
9305
|
+
// A conversation agent's waits are recorded and held as a resident's are
|
|
9306
|
+
// (nonOwningProviderWait), and its retries and in-turn restarts reach this
|
|
9307
|
+
// boundary without the scheduler: a wait recorded meanwhile, by its own
|
|
9308
|
+
// auxiliary call or a non-retryable failure, binds them here too. An
|
|
9309
|
+
// ephemeral run records nothing, so nothing binds it here. A frame whose
|
|
9310
|
+
// agent was replaced or removed (a fork disposed while its retry slept)
|
|
9311
|
+
// installs no hold: its turn is dropped after the compile, as main drops
|
|
9312
|
+
// any such turn.
|
|
9313
|
+
if (!this.ephemeralRuns.has(agent.name) && this.agents.get(agent.name) === agent) {
|
|
9314
|
+
const wait = this.providerWaits?.active(agent.name, agent.model);
|
|
9315
|
+
if (wait && this.holdForProviderWait(agent, wait, trigger ? [trigger] : [])) {
|
|
9316
|
+
if (ownsProviderGate)
|
|
9317
|
+
this.releasePrimaryProviderGate(agent.name);
|
|
9318
|
+
// A retry or an in-turn restart arrives with its turn still open at
|
|
9319
|
+
// the event gate: the failed attempt, or the predecessor that kept
|
|
9320
|
+
// liveness for it, leaves closing it to this frame. Held here, no
|
|
9321
|
+
// frame would close it, and while it is open every gate event
|
|
9322
|
+
// buffers, for every agent, until the wait ends.
|
|
9323
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
9324
|
+
console.error(`[provider-wait] agent=${agent.name} model=${agent.model} binds at admission ` +
|
|
9325
|
+
`(recorded after the wake was scheduled); wake held, not started`);
|
|
9326
|
+
return;
|
|
9327
|
+
}
|
|
9328
|
+
}
|
|
8383
9329
|
// Mark the turn alive before ANYTHING awaits (hooks, compile, stream
|
|
8384
9330
|
// setup): from here until this turn's teardown, cross-turn writers defer
|
|
8385
9331
|
// (addMessage guard) instead of appending — nothing may enter the window
|
|
@@ -8728,6 +9674,28 @@ export class AgentFramework {
|
|
|
8728
9674
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
8729
9675
|
return false;
|
|
8730
9676
|
}
|
|
9677
|
+
// A conversation agent takes no provider gate, so its own compression
|
|
9678
|
+
// can record a wait while this turn compiles, after the last boundary
|
|
9679
|
+
// checked. Check its waits again here, where nothing has been sent, and
|
|
9680
|
+
// hold the turn as the boundary would rather than send it into the wait.
|
|
9681
|
+
if (this.conversationAgentHomes.has(agent.name)) {
|
|
9682
|
+
const wait = this.providerWaits?.active(agent.name, agent.model);
|
|
9683
|
+
if (wait && this.holdForProviderWait(agent, wait, trigger ? [trigger] : [])) {
|
|
9684
|
+
console.error(`[provider-wait] agent=${agent.name} model=${agent.model} binds after the compile ` +
|
|
9685
|
+
`(recorded while the turn compiled); turn held, nothing sent`);
|
|
9686
|
+
// The turn emitted inference:started before its compile; every turn
|
|
9687
|
+
// that does gets one terminal event, which listeners (a typing loop)
|
|
9688
|
+
// rely on, as the failure sites' holds emit inference:failed.
|
|
9689
|
+
this.emitTrace({ type: 'inference:failed', agentName: agent.name, error: providerWaitText(agent.name, agent.model, wait) });
|
|
9690
|
+
stream.cancel();
|
|
9691
|
+
agent.cancelStream();
|
|
9692
|
+
this.activeStreams.delete(agent.name);
|
|
9693
|
+
this.pendingAssistantBlocks.delete(agent.name);
|
|
9694
|
+
this.channelRegistry?.stopTyping();
|
|
9695
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
9696
|
+
return false;
|
|
9697
|
+
}
|
|
9698
|
+
}
|
|
8731
9699
|
const handle = this.driveStream(agent, stream, requestSnapshot, trigger, attempt, compiledRequest, ownsProviderGate, takeKvSubmission, drainKvSubmissionIds, new Set(tools.map((tool) => tool.name)));
|
|
8732
9700
|
this.activeStreams.set(agent.name, handle);
|
|
8733
9701
|
// Handoff: driveStream captured the token in its synchronous prefix;
|
|
@@ -8753,9 +9721,27 @@ export class AgentFramework {
|
|
|
8753
9721
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
8754
9722
|
return false;
|
|
8755
9723
|
}
|
|
8756
|
-
const
|
|
9724
|
+
const statedWait = ownsProviderGate ? undefined : this.nonOwningProviderWait(agent, err, trigger);
|
|
9725
|
+
if (statedWait === 'held') {
|
|
9726
|
+
// A conversation agent held by the provider's stated wait, as a
|
|
9727
|
+
// resident is: the hold's release owns the later compile.
|
|
9728
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
9729
|
+
return false;
|
|
9730
|
+
}
|
|
9731
|
+
const action = statedWait === 'fail' ? { retry: false } : this.errorPolicy.onInferenceError(err, agent.name, attempt);
|
|
8757
9732
|
if (action.retry) {
|
|
8758
|
-
|
|
9733
|
+
// A policy's delay can be a provider's retry-after: wait all of it.
|
|
9734
|
+
await this.waitForRetry(action.delayMs);
|
|
9735
|
+
if (this.providerAdmissionClosed) {
|
|
9736
|
+
// stop() ended the wait: that cancels the retry, it does not hasten
|
|
9737
|
+
// it. Nothing is written (the store is closing); this frame still
|
|
9738
|
+
// owns its token, since no retry frame replaced it.
|
|
9739
|
+
if (this.activeTurnTokens.get(agent.name) === turnToken) {
|
|
9740
|
+
this.activeTurnTokens.delete(agent.name);
|
|
9741
|
+
this.activeTurnTriggers.delete(agent.name);
|
|
9742
|
+
}
|
|
9743
|
+
return false;
|
|
9744
|
+
}
|
|
8759
9745
|
// The retry re-enters startAgentStream, which replaces this frame's
|
|
8760
9746
|
// turn token with its own; cleanup belongs to the innermost frame.
|
|
8761
9747
|
await this.startAgentStream(agent, trigger, attempt + 1);
|
|
@@ -8773,16 +9759,18 @@ export class AgentFramework {
|
|
|
8773
9759
|
this.settleAgent(agent.name, {
|
|
8774
9760
|
stopReason: 'exhausted',
|
|
8775
9761
|
speech: '',
|
|
8776
|
-
error: err.message,
|
|
9762
|
+
error: statedWait === 'fail' ? this.statedWaitNotSlept(err) : err.message,
|
|
8777
9763
|
}, startLogical);
|
|
8778
9764
|
this.emitTrace({
|
|
8779
9765
|
type: 'inference:exhausted',
|
|
8780
9766
|
agentName: agent.name,
|
|
8781
9767
|
error: err.message,
|
|
8782
|
-
// Drives the poison-history breaker (
|
|
8783
|
-
//
|
|
8784
|
-
//
|
|
8785
|
-
//
|
|
9768
|
+
// Drives the poison-history breaker (shedMayHelp: an
|
|
9769
|
+
// `invalid_request` at HTTP 400, 413 or 422 or with no status, or a
|
|
9770
|
+
// `context_length` at 413, suggests the provider refused the
|
|
9771
|
+
// request's content or size, which a shed of the newest exchange may
|
|
9772
|
+
// fix; auth/abort and other context_length failures are also
|
|
9773
|
+
// non-retryable but must NOT cost history) and the OverBudget drain breaker
|
|
8786
9774
|
// (`over_budget`: compile refused to fit the hard budget — this is
|
|
8787
9775
|
// the site that sees it, since compile runs before the stream
|
|
8788
9776
|
// exists). `retryable` is kept for observability.
|
|
@@ -9374,7 +10362,7 @@ export class AgentFramework {
|
|
|
9374
10362
|
if (readyState.status === 'ready') {
|
|
9375
10363
|
const cap = this.resolveToolResultInlineCap(agent).cap;
|
|
9376
10364
|
const { blocks: toolResultContent, spilled } = await this.buildStoredToolResultContent(agent.name, readyState.toolResults, cap);
|
|
9377
|
-
agent.toolResultGuard.storeResults(toolResultContent, readyState.toolResults.map((tc) => this.toMembraneToolResult(tc.id, tc.result, cap, spilled.get(tc.id))), readyState.toolResults, this.silentTurnRowMetadata(agent.name));
|
|
10365
|
+
agent.toolResultGuard.storeResults(toolResultContent, readyState.toolResults.map((tc) => this.toMembraneToolResult(tc.id, tc.result, cap, spilled.get(tc.id))), readyState.toolResults, this.silentTurnRowMetadata(agent.name), cap);
|
|
9378
10366
|
// The response is already complete: this batch is never
|
|
9379
10367
|
// submitted in this turn, so it cannot be refused. Admit it.
|
|
9380
10368
|
agent.toolResultGuard.settleTurnEnded();
|
|
@@ -9597,7 +10585,7 @@ export class AgentFramework {
|
|
|
9597
10585
|
// border is close, rather than noise on every near-threshold
|
|
9598
10586
|
// flip. (It must not re-enter any agent's context — the emoji
|
|
9599
10587
|
// is suppressed at the surface via
|
|
9600
|
-
//
|
|
10588
|
+
// DISCORD_SUPPRESSED_REACTIONS_BASELINE; without that, marking a
|
|
9601
10589
|
// refusal feeds the next one.)
|
|
9602
10590
|
if (!handledByRewind) {
|
|
9603
10591
|
void this.reactToRefusal(agent.name, category);
|
|
@@ -9869,25 +10857,39 @@ export class AgentFramework {
|
|
|
9869
10857
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
9870
10858
|
break;
|
|
9871
10859
|
}
|
|
9872
|
-
const
|
|
10860
|
+
const statedWait = ownsProviderGate ? undefined : this.nonOwningProviderWait(agent, err, trigger);
|
|
10861
|
+
if (statedWait === 'held') {
|
|
10862
|
+
lifecyclePhase = 'failed';
|
|
10863
|
+
if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId)
|
|
10864
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
10865
|
+
break;
|
|
10866
|
+
}
|
|
10867
|
+
const action = statedWait === 'fail' ? { retry: false } : this.errorPolicy.onInferenceError(err, agent.name, attempt);
|
|
9873
10868
|
if (action.retry) {
|
|
9874
|
-
|
|
9875
|
-
await this.
|
|
10869
|
+
// A policy's delay can be a provider's retry-after: wait all of it.
|
|
10870
|
+
await this.waitForRetry(action.delayMs);
|
|
10871
|
+
// stop() ending the wait cancels the retry (this frame's finally
|
|
10872
|
+
// clears the token); it is never permission to retry at once.
|
|
10873
|
+
if (!this.providerAdmissionClosed)
|
|
10874
|
+
await this.startAgentStream(agent, trigger, attempt + 1);
|
|
9876
10875
|
}
|
|
9877
10876
|
else {
|
|
9878
10877
|
this.settleAgent(agent.name, {
|
|
9879
10878
|
stopReason: 'exhausted',
|
|
9880
10879
|
speech: '',
|
|
9881
|
-
error: err.message,
|
|
10880
|
+
error: statedWait === 'fail' ? this.statedWaitNotSlept(err) : err.message,
|
|
9882
10881
|
}, this.agents.get(agent.name) === agent && agent.streamId === myStreamId ? streamRouting.logical : undefined);
|
|
9883
10882
|
this.emitTrace({
|
|
9884
10883
|
type: 'inference:exhausted',
|
|
9885
10884
|
agentName: agent.name,
|
|
9886
10885
|
error: err.message,
|
|
9887
10886
|
// Drives the poison-history breaker. `retryable` is kept for
|
|
9888
|
-
// observability, but the
|
|
9889
|
-
//
|
|
9890
|
-
//
|
|
10887
|
+
// observability, but the breaker gates on
|
|
10888
|
+
// shedMayHelp(errorType, httpStatus): only an `invalid_request`
|
|
10889
|
+
// at HTTP 400, 413 or 422 or with no status, or a
|
|
10890
|
+
// `context_length` at 413, suggests the provider refused the
|
|
10891
|
+
// request's content or size, which a shed of the newest
|
|
10892
|
+
// exchange may fix.
|
|
9891
10893
|
...this.classifyInferenceError(err),
|
|
9892
10894
|
});
|
|
9893
10895
|
if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId)
|
|
@@ -10102,6 +11104,23 @@ export class AgentFramework {
|
|
|
10102
11104
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
10103
11105
|
return;
|
|
10104
11106
|
}
|
|
11107
|
+
// The same rule for a turn that doesn't own provider admission as at
|
|
11108
|
+
// the other two failure sites: a conversation agent's stated wait is
|
|
11109
|
+
// recorded and its turn held; an ephemeral run whose wait it won't sleep
|
|
11110
|
+
// says why it ends.
|
|
11111
|
+
const statedWait = ownsProviderGate ? undefined : this.nonOwningProviderWait(agent, err, trigger);
|
|
11112
|
+
if (statedWait === 'held') {
|
|
11113
|
+
this.emitTrace({ type: 'inference:failed', agentName: agent.name, error: err.message, stack: err.stack });
|
|
11114
|
+
this.logInference({ timestamp: startTime, agentName: agent.name, requestId, success: false,
|
|
11115
|
+
error: `Provider wait: ${err.message}`,
|
|
11116
|
+
request: compiledRequest ?? { note: 'streaming request rate-limited' }, durationMs });
|
|
11117
|
+
this.abortAgentScript(agent.name, 'provider wait');
|
|
11118
|
+
lifecyclePhase = 'failed';
|
|
11119
|
+
agent.reset();
|
|
11120
|
+
if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId)
|
|
11121
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
11122
|
+
return;
|
|
11123
|
+
}
|
|
10105
11124
|
this.emitTrace({
|
|
10106
11125
|
type: 'inference:failed',
|
|
10107
11126
|
agentName: agent.name,
|
|
@@ -10111,7 +11130,7 @@ export class AgentFramework {
|
|
|
10111
11130
|
this.settleAgent(agent.name, {
|
|
10112
11131
|
stopReason: 'exhausted',
|
|
10113
11132
|
speech: '',
|
|
10114
|
-
error: err.message,
|
|
11133
|
+
error: statedWait === 'fail' ? this.statedWaitNotSlept(err) : err.message,
|
|
10115
11134
|
}, streamRouting.logical);
|
|
10116
11135
|
this.emitTrace({
|
|
10117
11136
|
type: 'inference:exhausted',
|
|
@@ -11428,6 +12447,10 @@ export class AgentFramework {
|
|
|
11428
12447
|
logInference(entry) {
|
|
11429
12448
|
// Store large request/response as blobs
|
|
11430
12449
|
const entryToStore = { ...entry };
|
|
12450
|
+
// The failure text is stored inline in every record; the request that
|
|
12451
|
+
// produced it is already stored separately as a blob.
|
|
12452
|
+
if (typeof entryToStore.error === 'string')
|
|
12453
|
+
entryToStore.error = boundFailureText(entryToStore.error);
|
|
11431
12454
|
// Blob threshold: 10KB - typical context-heavy requests exceed this
|
|
11432
12455
|
const BLOB_THRESHOLD = 10000;
|
|
11433
12456
|
// Both branches persist the JSON view, never the live object: a compiled
|
|
@@ -11931,7 +12954,10 @@ export class AgentFramework {
|
|
|
11931
12954
|
// without this the only durable record of a failed inference is a field in
|
|
11932
12955
|
// llm-calls.jsonl — invisible to operator, agent, and monitoring.
|
|
11933
12956
|
if (event.type === 'inference:exhausted') {
|
|
11934
|
-
this.noteInferenceExhausted(event.agentName ?? 'unknown', event.error ?? 'unknown error', event.retryable, event.errorType
|
|
12957
|
+
this.noteInferenceExhausted(event.agentName ?? 'unknown', event.error ?? 'unknown error', event.retryable, event.errorType, {
|
|
12958
|
+
httpStatus: event.httpStatus,
|
|
12959
|
+
providerErrorCode: event.providerErrorCode,
|
|
12960
|
+
});
|
|
11935
12961
|
}
|
|
11936
12962
|
else if (event.type === 'inference:completed') {
|
|
11937
12963
|
// A successful response — even mid-turn between tool calls — proves the
|
|
@@ -11949,7 +12975,28 @@ export class AgentFramework {
|
|
|
11949
12975
|
...event,
|
|
11950
12976
|
timestamp: Date.now(),
|
|
11951
12977
|
};
|
|
11952
|
-
|
|
12978
|
+
// Listeners (web UI, subagent observers, host loggers) receive bounded
|
|
12979
|
+
// failure text. An event's `error` and `stack` strings are whatever its
|
|
12980
|
+
// source's Error carried, which for a provider failure can be an echo of
|
|
12981
|
+
// the whole request.
|
|
12982
|
+
const bounded = traceEvent;
|
|
12983
|
+
if (typeof bounded.error === 'string')
|
|
12984
|
+
bounded.error = boundFailureText(bounded.error);
|
|
12985
|
+
if (typeof bounded.stack === 'string')
|
|
12986
|
+
bounded.stack = boundFailureText(bounded.stack);
|
|
12987
|
+
// An event goes to the subscriptions present when its delivery starts,
|
|
12988
|
+
// each one only if it is still active when its turn comes, as the DOM's
|
|
12989
|
+
// EventTarget delivers: a subscription made during delivery waits for the
|
|
12990
|
+
// next event, and one removed during delivery is skipped if its turn
|
|
12991
|
+
// hasn't come. Each onTrace call is a subscription of its own, so a
|
|
12992
|
+
// function subscribed twice is called twice. Iterating the live array let
|
|
12993
|
+
// an unsubscribe during delivery make the listener after the one being
|
|
12994
|
+
// called miss the event, since splice shifts the rest under the iterator.
|
|
12995
|
+
for (const registration of [...this.traceListeners]) {
|
|
12996
|
+
if (!registration.active)
|
|
12997
|
+
continue;
|
|
12998
|
+
// Called as a plain function, as before, so the registration isn't its `this`.
|
|
12999
|
+
const { listener } = registration;
|
|
11953
13000
|
try {
|
|
11954
13001
|
listener(traceEvent);
|
|
11955
13002
|
}
|
|
@@ -11972,15 +13019,18 @@ export class AgentFramework {
|
|
|
11972
13019
|
* inference, so this never causes a retry/wake loop.
|
|
11973
13020
|
* 3. escalation — after N consecutive failures the agent is hard-down;
|
|
11974
13021
|
* log that loudly (a repeated identical failure is the textbook signal).
|
|
11975
|
-
* 4. breaker — when
|
|
11976
|
-
*
|
|
11977
|
-
*
|
|
11978
|
-
*
|
|
11979
|
-
*
|
|
11980
|
-
*
|
|
11981
|
-
*
|
|
11982
|
-
*
|
|
11983
|
-
* the
|
|
13022
|
+
* 4. breaker — when a failure at or past the hard-down threshold suggests
|
|
13023
|
+
* the provider refused the request's content or size (shedMayHelp:
|
|
13024
|
+
* an `invalid_request` at HTTP 400, 413 or 422 or with no status, or a
|
|
13025
|
+
* `context_length` at 413, such as a provider's answer to a corrupted
|
|
13026
|
+
* tool_use/tool_result pairing or an oversized attachment), retrying
|
|
13027
|
+
* the same context is likely to fail the same way, and every new push
|
|
13028
|
+
* event would wake the agent onto that same history. At the hard-down
|
|
13029
|
+
* threshold, automatically quarantine: shed the newest complete
|
|
13030
|
+
* exchange (the same forced-rewind primitive `/unstick` uses —
|
|
13031
|
+
* shedNewestTurn never orphans a tool_use/thinking block) and retry,
|
|
13032
|
+
* bounded by the same rewind cap so the breaker can never eat the
|
|
13033
|
+
* whole history.
|
|
11984
13034
|
*/
|
|
11985
13035
|
/**
|
|
11986
13036
|
* Live health snapshot for /healthz and doctor tooling: gate state, queued
|
|
@@ -12005,11 +13055,14 @@ export class AgentFramework {
|
|
|
12005
13055
|
primaryPending: gate?.primaryPending ?? false,
|
|
12006
13056
|
auxiliaryInFlight: gate?.auxiliaryInFlight ?? 0,
|
|
12007
13057
|
auxiliaryDeferred: gate?.deferredAuxiliary ?? 0,
|
|
12008
|
-
|
|
13058
|
+
// Null while none, and while a provider wait holds until released.
|
|
13059
|
+
cooldownUntil: cooldown !== undefined && Number.isFinite(cooldown.until) ? cooldown.until : null,
|
|
12009
13060
|
cooldownReason: cooldown?.reason ?? null,
|
|
12010
13061
|
hostHold: cooldown?.hostHoldError !== undefined,
|
|
12011
13062
|
heldRequests: cooldown?.heldRequests.length ?? 0,
|
|
12012
13063
|
lastRecovery: this.providerAccelerationLastRecovery?.get(name) ?? null,
|
|
13064
|
+
/** Provider waits (stated retry-after) binding this agent now, per model. */
|
|
13065
|
+
providerWaits: this.providerWaitSnapshot(name).map(({ agent: _agent, ...wait }) => wait),
|
|
12013
13066
|
};
|
|
12014
13067
|
})(),
|
|
12015
13068
|
lastInference: this.lastInferenceAt.get(name) ?? null,
|
|
@@ -12024,8 +13077,12 @@ export class AgentFramework {
|
|
|
12024
13077
|
* and kind-specific fields (additive only — doctor parses by regex). */
|
|
12025
13078
|
logFailure(record) {
|
|
12026
13079
|
try {
|
|
13080
|
+
// Every field is bounded: `reason` is conventionally free text, and any
|
|
13081
|
+
// caller's field may carry an error message, or an Error whose own
|
|
13082
|
+
// fields include the whole request.
|
|
13083
|
+
const bounded = boundFailureFields(record);
|
|
12027
13084
|
mkdirSync('logs', { recursive: true });
|
|
12028
|
-
appendFileSync('logs/failures.log', JSON.stringify({ at: new Date().toISOString(), ...
|
|
13085
|
+
appendFileSync('logs/failures.log', JSON.stringify({ at: new Date().toISOString(), ...bounded }) + '\n');
|
|
12029
13086
|
}
|
|
12030
13087
|
catch { /* best-effort */ }
|
|
12031
13088
|
}
|
|
@@ -12048,11 +13105,25 @@ export class AgentFramework {
|
|
|
12048
13105
|
notifyOps(kind, agentName, message, data) {
|
|
12049
13106
|
this.opsAlert(kind, agentName, message, { data });
|
|
12050
13107
|
}
|
|
12051
|
-
opsAlert(kind, agentName,
|
|
13108
|
+
opsAlert(kind, agentName, rawMessage, opts) {
|
|
13109
|
+
// A JavaScript caller of notifyOps or notifyOpsAlert can pass a message
|
|
13110
|
+
// that isn't a string; it's described as a thrown value is, rather than
|
|
13111
|
+
// thrown on (a string is its own description).
|
|
13112
|
+
const wholeMessage = failureDescription(rawMessage);
|
|
13113
|
+
const message = boundFailureText(wholeMessage);
|
|
13114
|
+
// The caller's data goes to failures.log and to trace listeners, bounded
|
|
13115
|
+
// the same way for both. A JavaScript caller's null is no data, and data
|
|
13116
|
+
// that isn't a plain object (a string, an array, a Buffer) is kept as one
|
|
13117
|
+
// field, `data`, rather than spread into a field per character or element.
|
|
13118
|
+
const rawData = opts?.data;
|
|
13119
|
+
const plain = typeof rawData === 'object' && rawData !== null &&
|
|
13120
|
+
[Object.prototype, null].includes(Object.getPrototypeOf(rawData));
|
|
13121
|
+
const data = rawData == null ? undefined
|
|
13122
|
+
: boundFailureFields(plain ? rawData : { data: rawData });
|
|
12052
13123
|
if (!opts?.skipLog) {
|
|
12053
|
-
this.logFailure({ agent: agentName, kind, reason: message, ...
|
|
13124
|
+
this.logFailure({ agent: agentName, kind, reason: message, ...data });
|
|
12054
13125
|
}
|
|
12055
|
-
this.emitTrace({ type: 'ops:alert', kind, agentName, message, data
|
|
13126
|
+
this.emitTrace({ type: 'ops:alert', kind, agentName, message, data });
|
|
12056
13127
|
const hook = process.env.CONNECTOME_OPS_WEBHOOK;
|
|
12057
13128
|
if (!hook)
|
|
12058
13129
|
return;
|
|
@@ -12066,7 +13137,7 @@ export class AgentFramework {
|
|
|
12066
13137
|
method: 'POST',
|
|
12067
13138
|
headers: { 'content-type': 'application/json' },
|
|
12068
13139
|
body: JSON.stringify({
|
|
12069
|
-
content: `\u{1F6A8} **${agentName}** ${kind}: ${
|
|
13140
|
+
content: `\u{1F6A8} **${agentName}** ${kind}: ${boundFailureText(wholeMessage, OPS_WEBHOOK_MESSAGE_MAX_CHARS)}`,
|
|
12070
13141
|
}),
|
|
12071
13142
|
}).catch((err) => console.error(`[ops-alert] webhook post failed (${key}):`, err instanceof Error ? err.message : err));
|
|
12072
13143
|
}
|
|
@@ -12103,9 +13174,12 @@ export class AgentFramework {
|
|
|
12103
13174
|
}
|
|
12104
13175
|
/**
|
|
12105
13176
|
* Classify a terminal inference error for the `inference:exhausted` trace.
|
|
12106
|
-
* Single-sourced for every emit site
|
|
12107
|
-
*
|
|
12108
|
-
*
|
|
13177
|
+
* Single-sourced for every emit site that has an error to classify (a
|
|
13178
|
+
* stream the framework aborted has none): the downstream breakers gate on
|
|
13179
|
+
* what it returns (the poison-history quarantine on `errorType` and
|
|
13180
|
+
* `httpStatus`, through shedMayHelp; the OverBudget drain kick on
|
|
13181
|
+
* `errorType`), so classification drift between sites would silently
|
|
13182
|
+
* change when a safety net acts.
|
|
12109
13183
|
*
|
|
12110
13184
|
* context-manager's refusal errors are matched by a real `instanceof` since
|
|
12111
13185
|
* CM exports them from its package root (context-manager#41 / #71; floor
|
|
@@ -12117,8 +13191,18 @@ export class AgentFramework {
|
|
|
12117
13191
|
* can be reworded without warning.
|
|
12118
13192
|
*/
|
|
12119
13193
|
classifyInferenceError(err) {
|
|
12120
|
-
|
|
12121
|
-
|
|
13194
|
+
// From either copy of membrane (isMembraneError). The fields are read as
|
|
13195
|
+
// data, and the type is bounded like describeFailure's, for every sink the
|
|
13196
|
+
// classification reaches (the trace, the marker, the logs).
|
|
13197
|
+
if (isMembraneError(err)) {
|
|
13198
|
+
const fields = err;
|
|
13199
|
+
const providerErrorCode = boundProviderErrorCode(fields.providerErrorCode);
|
|
13200
|
+
return {
|
|
13201
|
+
...(typeof fields.retryable === 'boolean' ? { retryable: fields.retryable } : {}),
|
|
13202
|
+
...(typeof fields.type === 'string' ? { errorType: boundFailureText(fields.type, PROVIDER_ERROR_CODE_MAX_CHARS) } : {}),
|
|
13203
|
+
...(typeof fields.httpStatus === 'number' ? { httpStatus: fields.httpStatus } : {}),
|
|
13204
|
+
...(providerErrorCode !== undefined ? { providerErrorCode } : {}),
|
|
13205
|
+
};
|
|
12122
13206
|
}
|
|
12123
13207
|
if (err instanceof OverBudgetError || err.name === 'OverBudgetError') {
|
|
12124
13208
|
return { errorType: 'over_budget' };
|
|
@@ -12142,31 +13226,72 @@ export class AgentFramework {
|
|
|
12142
13226
|
notifyOpsAlert(kind, agentName, message, data) {
|
|
12143
13227
|
this.opsAlert(kind, agentName, message, { data });
|
|
12144
13228
|
}
|
|
12145
|
-
noteInferenceExhausted(agentName,
|
|
13229
|
+
noteInferenceExhausted(agentName, rawReason, retryable, rawErrorType, detail) {
|
|
13230
|
+
// The reason is the failure's own message, and a provider can echo the
|
|
13231
|
+
// whole rejected request in it: one household's [inference-failed]
|
|
13232
|
+
// marker carried ~1.7 MB of its own request back into its context. Every
|
|
13233
|
+
// sink below gets an excerpt cut from the whole reason at its own bound
|
|
13234
|
+
// (boundFailureText, so its marker states the whole length): the stderr
|
|
13235
|
+
// line, failures.log and the hard-down alert FAILURE_TEXT_MAX_CHARS, the
|
|
13236
|
+
// [inference-failed] marker FAILURE_MARKER_REASON_MAX_CHARS, the status's
|
|
13237
|
+
// lastError and the compile-refused alert FAILURE_SUMMARY_MAX_CHARS, and
|
|
13238
|
+
// the rewind marker REWIND_MARKER_REASON_MAX_CHARS. The classification
|
|
13239
|
+
// travels as structured fields beside the text.
|
|
13240
|
+
//
|
|
13241
|
+
// A reason that isn't a string (the stream paths always pass a message)
|
|
13242
|
+
// is described rather than allowed to throw before anything is recorded.
|
|
13243
|
+
// Its whole description is the whole reason, cut once by each sink.
|
|
13244
|
+
const wholeReason = typeof rawReason === 'string' ? rawReason : failureDescription(rawReason);
|
|
13245
|
+
// The type is bounded wherever it's written, as classifyInferenceError
|
|
13246
|
+
// bounds the types it reads, since a caller can pass one it didn't classify.
|
|
13247
|
+
const errorType = typeof rawErrorType === 'string'
|
|
13248
|
+
? boundFailureText(rawErrorType, PROVIDER_ERROR_CODE_MAX_CHARS) : rawErrorType;
|
|
13249
|
+
const reason = boundFailureText(wholeReason);
|
|
13250
|
+
const markerReason = boundFailureText(wholeReason, FAILURE_MARKER_REASON_MAX_CHARS);
|
|
13251
|
+
const httpStatus = typeof detail?.httpStatus === 'number' ? detail.httpStatus : undefined;
|
|
13252
|
+
const providerErrorCode = boundProviderErrorCode(detail?.providerErrorCode);
|
|
13253
|
+
const classification = {
|
|
13254
|
+
...(errorType !== undefined ? { errorType } : {}),
|
|
13255
|
+
...(retryable !== undefined ? { retryable } : {}),
|
|
13256
|
+
...(httpStatus !== undefined ? { httpStatus } : {}),
|
|
13257
|
+
...(providerErrorCode !== undefined ? { providerErrorCode } : {}),
|
|
13258
|
+
};
|
|
13259
|
+
// Each excerpt says how long the whole was when it was cut.
|
|
13260
|
+
const cutAt = (excerpt) => excerpt === wholeReason ? {} : { reasonChars: wholeReason.length };
|
|
12146
13261
|
const streak = (this.consecutiveInferenceFailures.get(agentName) ?? 0) + 1;
|
|
12147
13262
|
this.consecutiveInferenceFailures.set(agentName, streak);
|
|
12148
|
-
this.lastInferenceAt.set(agentName, { ...this.lastInferenceAt.get(agentName), failedAt: Date.now(), lastError:
|
|
13263
|
+
this.lastInferenceAt.set(agentName, { ...this.lastInferenceAt.get(agentName), failedAt: Date.now(), lastError: boundFailureText(wholeReason, FAILURE_SUMMARY_MAX_CHARS) });
|
|
12149
13264
|
// (1) Durable stderr line — works in headless/daemon mode with no client.
|
|
12150
|
-
|
|
13265
|
+
const labels = [
|
|
13266
|
+
errorType !== undefined ? `type=${errorType}` : undefined,
|
|
13267
|
+
httpStatus !== undefined ? `status=${httpStatus}` : undefined,
|
|
13268
|
+
providerErrorCode !== undefined ? `code=${providerErrorCode}` : undefined,
|
|
13269
|
+
].filter((label) => label !== undefined);
|
|
13270
|
+
console.error(`[inference-failed] agent=${agentName} consecutive=${streak}` +
|
|
13271
|
+
`${labels.length > 0 ? ` ${labels.join(' ')}` : ''}: ${reason}`);
|
|
12151
13272
|
// (1b) Machine-greppable durable record, independent of journald/unit log
|
|
12152
13273
|
// redirects: logs/failures.log under the host's working directory. This is
|
|
12153
|
-
// what connectome-doctor reads. Legacy fields kept; `kind`
|
|
12154
|
-
|
|
13274
|
+
// what connectome-doctor reads. Legacy fields kept; `kind` and the
|
|
13275
|
+
// classification fields are additive.
|
|
13276
|
+
this.logFailure({ agent: agentName, consecutive: streak, reason, kind: 'inference-exhausted', ...classification, ...cutAt(reason) });
|
|
12155
13277
|
// (2) Agent-facing chronicle marker (no inference triggered → no loop).
|
|
12156
13278
|
const agent = this.agents.get(agentName);
|
|
12157
13279
|
if (agent && process.env.SUPPRESS_INFERENCE_FAILED_MARKER !== '1') {
|
|
13280
|
+
const cause = [errorType, httpStatus !== undefined ? `HTTP ${httpStatus}` : undefined]
|
|
13281
|
+
.filter((part) => part !== undefined)
|
|
13282
|
+
.join(', ');
|
|
12158
13283
|
try {
|
|
12159
13284
|
agent.getContextManager().addMessage('user', [{
|
|
12160
13285
|
type: 'text',
|
|
12161
13286
|
text: `[inference-failed] Your previous turn did not complete: the model ` +
|
|
12162
|
-
`call failed and produced no response, so nothing was sent. Reason: ` +
|
|
12163
|
-
`${
|
|
13287
|
+
`call failed${cause ? ` (${cause})` : ''} and produced no response, so nothing was sent. Reason: ` +
|
|
13288
|
+
`${markerReason}. If this recurs with the same cause, change approach ` +
|
|
12164
13289
|
`rather than retrying identically (e.g. drop an oversized attachment ` +
|
|
12165
13290
|
`or an unsupported setting).`,
|
|
12166
|
-
}], { system: true, kind: 'inference-failed', reason, consecutive: streak });
|
|
13291
|
+
}], { system: true, kind: 'inference-failed', reason: markerReason, consecutive: streak, ...classification, ...cutAt(markerReason) });
|
|
12167
13292
|
}
|
|
12168
13293
|
catch (err) {
|
|
12169
|
-
console.error(`[inference-failed] could not record chronicle marker for ${agentName}
|
|
13294
|
+
console.error(`[inference-failed] could not record chronicle marker for ${agentName}: ${describeFailureWithFrames(err)}`);
|
|
12170
13295
|
}
|
|
12171
13296
|
}
|
|
12172
13297
|
// (2b) OverBudget deadlock breaker. When compile fails with
|
|
@@ -12184,16 +13309,22 @@ export class AgentFramework {
|
|
|
12184
13309
|
// (e.g. a reason string that crossed a serialization boundary). It matches
|
|
12185
13310
|
// CM's current OverBudgetError wording and MAY rot if CM rewords it — the
|
|
12186
13311
|
// errorType gate is the one that's load-bearing.
|
|
13312
|
+
//
|
|
13313
|
+
// The fallback applies only to an unclassified reason. A classified
|
|
13314
|
+
// failure (a provider's 400, say) already says what it is, and its
|
|
13315
|
+
// message is the provider's text, which can quote anything, including
|
|
13316
|
+
// these phrases from the agent's own context.
|
|
12187
13317
|
const overBudget = errorType === 'over_budget' ||
|
|
12188
13318
|
errorType === 'context_refusal' ||
|
|
12189
|
-
|
|
13319
|
+
// Decided on the whole reason: presentation bounds never change it.
|
|
13320
|
+
(errorType === undefined && /exceed hard budget|no summary covers/i.test(wholeReason));
|
|
12190
13321
|
// Observability: a refused compile means the agent cannot think AT ALL
|
|
12191
13322
|
// this turn — surface it on the ops-alert pipeline (ops:alert trace +
|
|
12192
13323
|
// webhook → fleet-watch) immediately, not only at the hard-down streak.
|
|
12193
13324
|
// The per-(agent,kind) cooldown throttles repeats; failures.log already
|
|
12194
13325
|
// has the per-exhaustion record from logFailure above.
|
|
12195
13326
|
if (overBudget) {
|
|
12196
|
-
this.opsAlert('context-refusal', agentName, `compile refused (${errorType ?? 'unclassified'}): ${
|
|
13327
|
+
this.opsAlert('context-refusal', agentName, `compile refused (${errorType ?? 'unclassified'}): ${boundFailureText(wholeReason, FAILURE_SUMMARY_MAX_CHARS)} — ` +
|
|
12197
13328
|
`drain breaker ${agent ? 'kicking' : 'unavailable (no agent handle)'}; ` +
|
|
12198
13329
|
`if this persists, raise contextBudgetTokens or overBudgetGraceRatio.`, { skipLog: true });
|
|
12199
13330
|
}
|
|
@@ -12210,7 +13341,7 @@ export class AgentFramework {
|
|
|
12210
13341
|
console.error(`[inference-failed] drain kicked for ${agentName} (OverBudget breaker, ${ticks} ticks)`);
|
|
12211
13342
|
}
|
|
12212
13343
|
catch (err) {
|
|
12213
|
-
console.error(`[inference-failed] drain kick failed for ${agentName} after ${ticks} ticks
|
|
13344
|
+
console.error(`[inference-failed] drain kick failed for ${agentName} after ${ticks} ticks: ${describeFailureWithFrames(err)}`);
|
|
12214
13345
|
}
|
|
12215
13346
|
finally {
|
|
12216
13347
|
this.overBudgetDrainInFlight.delete(agentName);
|
|
@@ -12224,18 +13355,26 @@ export class AgentFramework {
|
|
|
12224
13355
|
// Ops alert (webhook + ops:alert trace), throttled per (agent, kind) —
|
|
12225
13356
|
// a hard-down agent re-posts every ~15 min, not on every failed retry.
|
|
12226
13357
|
// failures.log already got the per-exhaustion record above.
|
|
12227
|
-
|
|
12228
|
-
//
|
|
12229
|
-
|
|
12230
|
-
|
|
12231
|
-
//
|
|
12232
|
-
//
|
|
12233
|
-
//
|
|
12234
|
-
//
|
|
12235
|
-
//
|
|
13358
|
+
// The excerpt leaves room for the prefix, so the alert's own bound
|
|
13359
|
+
// never cuts it a second time.
|
|
13360
|
+
const hardDown = `${streak} consecutive inference failures. Last reason: `;
|
|
13361
|
+
this.opsAlert('hard-down', agentName, hardDown + boundFailureText(wholeReason, FAILURE_TEXT_MAX_CHARS - hardDown.length), { skipLog: true });
|
|
13362
|
+
// (4) Poison-history breaker: ONLY when shedMayHelp(errorType,
|
|
13363
|
+
// httpStatus) holds, for an `invalid_request` at HTTP 400, 413 or 422 or
|
|
13364
|
+
// with no status, or a `context_length` at 413, which suggests the
|
|
13365
|
+
// provider refused the request's content or size. The current failure
|
|
13366
|
+
// decides, whatever failed before it in the streak. Deliberately NOT
|
|
13367
|
+
// keyed on `retryable === false`, which membrane also returns for auth
|
|
13368
|
+
// (expired key), abort (deliberate cancel), a context_length at 400 or
|
|
13369
|
+
// 422 (a token overflow, which the context budget exists to prevent;
|
|
13370
|
+
// shedding the newest exchange is the wrong direction for it), safety
|
|
13371
|
+
// and unsupported. For none of them is shedding the newest exchange
|
|
13372
|
+
// likely to help, and each would otherwise shed good exchanges and
|
|
13373
|
+
// stamp a rewind marker saying the API kept refusing them.
|
|
13374
|
+
// `retryable` is kept only for the trace.
|
|
12236
13375
|
void retryable;
|
|
12237
|
-
if (
|
|
12238
|
-
this.quarantinePoisonedHistory(agent,
|
|
13376
|
+
if (agent && shedMayHelp(errorType, httpStatus)) {
|
|
13377
|
+
this.quarantinePoisonedHistory(agent, wholeReason);
|
|
12239
13378
|
}
|
|
12240
13379
|
}
|
|
12241
13380
|
}
|
|
@@ -12245,14 +13384,14 @@ export class AgentFramework {
|
|
|
12245
13384
|
* primitives as the refusal auto-rewind / `/unstick` (shedNewestTurn +
|
|
12246
13385
|
* the single consolidated episode marker) — and queues a retry so the
|
|
12247
13386
|
* rewound history is verified immediately instead of waiting for the next
|
|
12248
|
-
* push event to wake the agent onto the same
|
|
13387
|
+
* push event to wake the agent onto the same context.
|
|
12249
13388
|
*
|
|
12250
13389
|
* Bounded: at most `refusalHandling.maxRewinds` (default 3, hard cap 10)
|
|
12251
13390
|
* sheds per failure episode; the budget resets on any successful inference.
|
|
12252
13391
|
* At the cap (or with nothing left to shed) it stops — the agent stays up
|
|
12253
13392
|
* and hard-down logging continues, but no further history is consumed.
|
|
12254
13393
|
*/
|
|
12255
|
-
quarantinePoisonedHistory(agent,
|
|
13394
|
+
quarantinePoisonedHistory(agent, wholeReason) {
|
|
12256
13395
|
const cap = Math.max(1, Math.min(10, agent.refusalHandling?.maxRewinds ?? 3));
|
|
12257
13396
|
const used = this.exhaustionRewinds.get(agent.name) ?? 0;
|
|
12258
13397
|
if (used >= cap) {
|
|
@@ -12267,7 +13406,7 @@ export class AgentFramework {
|
|
|
12267
13406
|
return;
|
|
12268
13407
|
}
|
|
12269
13408
|
this.exhaustionRewinds.set(agent.name, used + 1);
|
|
12270
|
-
const count = this.updateRewindMarker(agent,
|
|
13409
|
+
const count = this.updateRewindMarker(agent, boundFailureText(wholeReason, REWIND_MARKER_REASON_MAX_CHARS), 'inference-failure');
|
|
12271
13410
|
console.error(`[inference-rewind] agent=${agent.name} auto-quarantined ${rec.descriptor} ` +
|
|
12272
13411
|
`(${count} turn(s) shed this episode, cap ${cap}) after non-retryable ` +
|
|
12273
13412
|
`inference failures — retrying on the rewound history.`);
|
|
@@ -12461,7 +13600,11 @@ export class AgentFramework {
|
|
|
12461
13600
|
// Discover tools from all connected servers
|
|
12462
13601
|
await this.refreshMcplTools();
|
|
12463
13602
|
}
|
|
12464
|
-
/**
|
|
13603
|
+
/**
|
|
13604
|
+
* Reconcile the durable ledger with Chronicle, then deliver every server's
|
|
13605
|
+
* work. A drain still running is waited for and then followed by a fresh
|
|
13606
|
+
* pass (drainForGeneration).
|
|
13607
|
+
*/
|
|
12465
13608
|
async syncDiscordAwarenessMarkers(_onlyServerId) {
|
|
12466
13609
|
if (!this.discordAwarenessOutbox)
|
|
12467
13610
|
return;
|
|
@@ -12629,34 +13772,66 @@ export class AgentFramework {
|
|
|
12629
13772
|
throw new DiscordAwarenessAccountingError('branch reconciliation', error);
|
|
12630
13773
|
}
|
|
12631
13774
|
const pending = this.readDiscordAwarenessPending();
|
|
12632
|
-
|
|
13775
|
+
// A drain that is still running holds operations it read before this
|
|
13776
|
+
// reconciliation, and it sends them whatever the ledger now says. A branch
|
|
13777
|
+
// move can hide them from `pending`: restoring a rollback's source makes
|
|
13778
|
+
// its marks undesired, and an add whose reply has not arrived is not yet
|
|
13779
|
+
// present, so the ledger shows nothing to do while the drain goes on to
|
|
13780
|
+
// send its saved adds and then the removals they need. So a generation
|
|
13781
|
+
// accounts for every running drain as well as for every pending
|
|
13782
|
+
// operation, and waits for those drains instead of releasing the data
|
|
13783
|
+
// planes beside them (drainForGeneration).
|
|
13784
|
+
const serverIds = [...new Set([
|
|
13785
|
+
...pending.map((operation) => operation.ref.serverId),
|
|
13786
|
+
...this.discordAwarenessDrains.keys(),
|
|
13787
|
+
])];
|
|
13788
|
+
if (serverIds.length === 0) {
|
|
12633
13789
|
return {
|
|
12634
13790
|
requiresBarrier: false,
|
|
12635
13791
|
promise: Promise.resolve({ status: 'delivered', delivered: 0, failed: 0 }),
|
|
12636
13792
|
};
|
|
12637
13793
|
}
|
|
12638
|
-
const
|
|
12639
|
-
const promise = Promise.all(serverIds.map((serverId) => this.drainDiscordAwarenessOutbox(serverId))).then((outcomes) => {
|
|
12640
|
-
if (outcomes.every((outcome) => outcome.status === 'unavailable')) {
|
|
12641
|
-
return {
|
|
12642
|
-
status: 'unavailable',
|
|
12643
|
-
accounted: outcomes.reduce((count, outcome) => count + (outcome.status === 'unavailable' ? outcome.accounted : 0), 0),
|
|
12644
|
-
};
|
|
12645
|
-
}
|
|
12646
|
-
return {
|
|
12647
|
-
status: 'delivered',
|
|
12648
|
-
delivered: outcomes.reduce((count, outcome) => count + (outcome.status === 'delivered' ? outcome.delivered : 0), 0),
|
|
12649
|
-
failed: outcomes.reduce((count, outcome) => count + (outcome.status === 'delivered' ? outcome.failed : 0), 0),
|
|
12650
|
-
};
|
|
12651
|
-
});
|
|
13794
|
+
const promise = Promise.all(serverIds.map((serverId) => this.drainForGeneration(serverId))).then(combineDrainOutcomes);
|
|
12652
13795
|
return { requiresBarrier: true, promise };
|
|
12653
13796
|
}
|
|
13797
|
+
/**
|
|
13798
|
+
* One server's drain for a new generation: a gate completes only after
|
|
13799
|
+
* every operation pending when it was armed has been attempted. A drain
|
|
13800
|
+
* already running can't promise that. It never re-sends a key it has
|
|
13801
|
+
* attempted, so an operation re-pended while it runs (a rollback, its undo,
|
|
13802
|
+
* then a redo, all mid-drain) stays pending, and it may already have made
|
|
13803
|
+
* its last read of the ledger, missing a batch activated since. So a
|
|
13804
|
+
* generation that finds a drain running waits for it, then drains again
|
|
13805
|
+
* with a fresh pass that reads the ledger anew. Two generations waiting on
|
|
13806
|
+
* the same drain share the fresh pass that follows it.
|
|
13807
|
+
*/
|
|
13808
|
+
drainForGeneration(serverId) {
|
|
13809
|
+
const running = this.discordAwarenessDrains.get(serverId);
|
|
13810
|
+
if (!running)
|
|
13811
|
+
return this.drainDiscordAwarenessOutbox(serverId);
|
|
13812
|
+
return running.then((first) => this.drainDiscordAwarenessOutbox(serverId)
|
|
13813
|
+
.then((fresh) => combineDrainOutcomes([first, fresh])));
|
|
13814
|
+
}
|
|
13815
|
+
/**
|
|
13816
|
+
* Wait out the awareness gate for an inbound event: the generation current
|
|
13817
|
+
* now, and any generation installed while waiting, until none is current
|
|
13818
|
+
* or the current one is one already waited for. A handler that waited only
|
|
13819
|
+
* for the generation it first saw would go on behind a newer one armed
|
|
13820
|
+
* meanwhile. Rejects as the generation it waited for rejects.
|
|
13821
|
+
*/
|
|
13822
|
+
async awaitDiscordAwarenessBarriers() {
|
|
13823
|
+
let waited = null;
|
|
13824
|
+
for (let barrier = this.discordAwarenessBarrier; barrier && barrier !== waited; barrier = this.discordAwarenessBarrier) {
|
|
13825
|
+
waited = barrier;
|
|
13826
|
+
await barrier.promise;
|
|
13827
|
+
}
|
|
13828
|
+
}
|
|
12654
13829
|
/**
|
|
12655
13830
|
* Install one framework-global Discord-awareness generation before releasing
|
|
12656
|
-
* any inference-bearing event. With pending work
|
|
12657
|
-
* paused while all control planes remain live.
|
|
12658
|
-
* runs synchronously so request responders preserve
|
|
12659
|
-
* same-stack behavior.
|
|
13831
|
+
* any inference-bearing event. With pending work or a drain still running,
|
|
13832
|
+
* every MCPL data plane is paused while all control planes remain live.
|
|
13833
|
+
* With neither, ready() runs synchronously so request responders preserve
|
|
13834
|
+
* their historical same-stack behavior.
|
|
12660
13835
|
*/
|
|
12661
13836
|
installMcplDataPlaneGate() {
|
|
12662
13837
|
for (const connection of this.mcplServerRegistry?.getAllServers() ?? []) {
|
|
@@ -12744,6 +13919,13 @@ export class AgentFramework {
|
|
|
12744
13919
|
if (!this.mcplServerRegistry || !this.mcplHostCapabilities) {
|
|
12745
13920
|
throw new Error('MCPL subsystem is not initialized');
|
|
12746
13921
|
}
|
|
13922
|
+
// Every connect needs a readable awareness ledger: a runtime connect's
|
|
13923
|
+
// awareness barrier reconciles it, and a stdio child taking the
|
|
13924
|
+
// protective default derives that from it. Read it first, so an
|
|
13925
|
+
// unreadable ledger refuses the connect before anything is registered
|
|
13926
|
+
// under its id or spawned (a reconnecting server would otherwise be left
|
|
13927
|
+
// registered as a closed stub). With no outbox configured nothing is read.
|
|
13928
|
+
this.getPlacedReactionBaseline();
|
|
12747
13929
|
// Register prefix + config for tool dispatch routing (idempotent with
|
|
12748
13930
|
// the pre-registration pass in initializeMcpl).
|
|
12749
13931
|
const prefix = config.toolPrefix ?? `mcpl--${config.id}`;
|
|
@@ -12754,7 +13936,31 @@ export class AgentFramework {
|
|
|
12754
13936
|
if (this.channelRegistry) {
|
|
12755
13937
|
this.channelRegistry.setSubscriptionPolicy(config.id, config.channelSubscription ?? 'manual');
|
|
12756
13938
|
}
|
|
12757
|
-
|
|
13939
|
+
// A stdio child's protective reaction default, DISCORD_SUPPRESSED_REACTIONS_BASELINE
|
|
13940
|
+
// (WebSocket servers own their environment outside this process). The
|
|
13941
|
+
// caller's config stays unchanged, so a runtime restart derives afresh.
|
|
13942
|
+
// The default is derived at each spawn: the env below is a getter, which
|
|
13943
|
+
// the transport reads at first connect, at every automatic reconnect and
|
|
13944
|
+
// at a stub's retry, so retained markers added since the last spawn are
|
|
13945
|
+
// covered.
|
|
13946
|
+
let connectionConfig = config;
|
|
13947
|
+
if (!isWebSocketTransport(config)) {
|
|
13948
|
+
const declared = this.declaredReactionBaseline(config);
|
|
13949
|
+
if (declared) {
|
|
13950
|
+
console.error(`[mcpl] ${config.id}: DISCORD_SUPPRESSED_REACTIONS_BASELINE ${declared.source === 'env' ? 'from its env' : 'inherited from the host'} ` +
|
|
13951
|
+
`replaces the framework's protective default`);
|
|
13952
|
+
}
|
|
13953
|
+
connectionConfig = { ...config };
|
|
13954
|
+
Object.defineProperty(connectionConfig, 'env', {
|
|
13955
|
+
enumerable: true,
|
|
13956
|
+
configurable: true,
|
|
13957
|
+
get: () => ({
|
|
13958
|
+
...config.env,
|
|
13959
|
+
DISCORD_SUPPRESSED_REACTIONS_BASELINE: declared?.value ?? this.getPlacedReactionBaseline().join(','),
|
|
13960
|
+
}),
|
|
13961
|
+
});
|
|
13962
|
+
}
|
|
13963
|
+
const connection = await this.mcplServerRegistry.addServer(connectionConfig, this.mcplHostCapabilities);
|
|
12758
13964
|
// Wire listeners before either startup staging or the runtime global gate
|
|
12759
13965
|
// releases control traffic needed for registration and marker service.
|
|
12760
13966
|
this.wireMcplEvents(connection);
|
|
@@ -13076,6 +14282,91 @@ export class AgentFramework {
|
|
|
13076
14282
|
// ==========================================================================
|
|
13077
14283
|
// Runtime MCPL server lifecycle (agent-facing hot deploy/restart/unload)
|
|
13078
14284
|
// ==========================================================================
|
|
14285
|
+
/**
|
|
14286
|
+
* Snapshot the protective Discord reaction default for this deployment, as
|
|
14287
|
+
* suppression tokens: the forms the Discord adapter's suppression matcher
|
|
14288
|
+
* reads, not marker payloads for an add, a remove or a replay. It covers
|
|
14289
|
+
* the reactions the framework places: refusal markers, the adapter's own
|
|
14290
|
+
* default awareness marker, the configured awareness marker, and retained
|
|
14291
|
+
* outbox markers (including offline recovery choices and marks that can
|
|
14292
|
+
* later be removed or replayed). A custom emoji appears as its bare ID,
|
|
14293
|
+
* which suppression matches even after a rename; stored markers and the
|
|
14294
|
+
* actual add/remove payloads stay unchanged. Explicit adapter or operator
|
|
14295
|
+
* suppression settings still take precedence. Throws on an unreadable
|
|
14296
|
+
* ledger rather than reporting a partial baseline.
|
|
14297
|
+
*/
|
|
14298
|
+
getPlacedReactionBaseline() {
|
|
14299
|
+
let retained;
|
|
14300
|
+
try {
|
|
14301
|
+
retained = this.discordAwarenessOutbox?.batches().map((batch) => batch.emoji) ?? [];
|
|
14302
|
+
}
|
|
14303
|
+
catch (error) {
|
|
14304
|
+
throw new DiscordAwarenessAccountingError('reaction-baseline ledger read', error);
|
|
14305
|
+
}
|
|
14306
|
+
// Discord events carry the custom emoji's name and ID separately. Full
|
|
14307
|
+
// marker forms belong in add/remove calls, but suppression matches their
|
|
14308
|
+
// stable ID (including after a rename), rather than the composite token.
|
|
14309
|
+
const suppressionToken = (emoji) => {
|
|
14310
|
+
const custom = /^(?:<a?:\w+:(\d{17,20})>|(?:a:)?\w+:(\d{17,20}))$/.exec(emoji.trim());
|
|
14311
|
+
return custom?.[1] ?? custom?.[2] ?? emoji;
|
|
14312
|
+
};
|
|
14313
|
+
return [...new Set([
|
|
14314
|
+
...REFUSAL_REACTION_BASELINE,
|
|
14315
|
+
DEFAULT_DISCORD_AWARENESS_EMOJI,
|
|
14316
|
+
this.discordAwarenessEmoji,
|
|
14317
|
+
...retained,
|
|
14318
|
+
].map(suppressionToken))].filter((emoji) => emoji.length > 0);
|
|
14319
|
+
}
|
|
14320
|
+
/**
|
|
14321
|
+
* The reaction baseline an operator chose for a stdio server, if any: a
|
|
14322
|
+
* value declared in its env, or one inherited from the host with
|
|
14323
|
+
* `inheritEnv`, the declared one first. A value with no tokens, as
|
|
14324
|
+
* discord-mcpl's parser reads them, counts as unset: the adapter splits on
|
|
14325
|
+
* commas and keeps a token only when its normalizer leaves something of it
|
|
14326
|
+
* (blank, `:`, `::` and a lone U+FE0F leave nothing), so such a value (an
|
|
14327
|
+
* empty systemd `Environment=`, say, or `${A},${B}` or `:${NAME}:` with the
|
|
14328
|
+
* variables empty) would otherwise turn the protective default off without
|
|
14329
|
+
* anyone choosing that.
|
|
14330
|
+
*
|
|
14331
|
+
* Windows names are matched in both sources: Workers expose a case-
|
|
14332
|
+
* sensitive `process.env` copy. Within one source, Node's spawn keeps the
|
|
14333
|
+
* first spelling in sorted order, so each source is resolved that way.
|
|
14334
|
+
*
|
|
14335
|
+
* A value is read as the child will see it. A config loaded from JSON can
|
|
14336
|
+
* carry a non-string there despite the type, and spawn writes every
|
|
14337
|
+
* defined value as its string form, so a number is a declared value, not a
|
|
14338
|
+
* failed connect. A null counts as unset, like a blank: written in JSON it
|
|
14339
|
+
* means no value, and read as spawn's "null" it would replace the
|
|
14340
|
+
* protective default with a list that suppresses nothing.
|
|
14341
|
+
*/
|
|
14342
|
+
declaredReactionBaseline(config) {
|
|
14343
|
+
// A token as discord-mcpl's parser keeps one: `parseSuppressionEnvTokens`
|
|
14344
|
+
// trims each comma-separated token and keeps it when
|
|
14345
|
+
// `normalizeReactionEmoji` (discord-mcpl `src/filters.ts`) leaves
|
|
14346
|
+
// something, after it strips U+FE0F, trims, and strips one leading and one
|
|
14347
|
+
// trailing colon. The same steps in the same order, so the two can be
|
|
14348
|
+
// compared when either changes.
|
|
14349
|
+
const isToken = (token) => token.trim().replace(/\uFE0F/g, '').trim().replace(/^:|:$/g, '') !== '';
|
|
14350
|
+
const valueIn = (env) => {
|
|
14351
|
+
let raw;
|
|
14352
|
+
if (process.platform !== 'win32')
|
|
14353
|
+
raw = env?.DISCORD_SUPPRESSED_REACTIONS_BASELINE;
|
|
14354
|
+
else {
|
|
14355
|
+
const key = Object.keys(env ?? {}).sort()
|
|
14356
|
+
.find((name) => name.toUpperCase() === 'DISCORD_SUPPRESSED_REACTIONS_BASELINE');
|
|
14357
|
+
raw = key === undefined ? undefined : env?.[key];
|
|
14358
|
+
}
|
|
14359
|
+
if (raw === undefined || raw === null)
|
|
14360
|
+
return undefined;
|
|
14361
|
+
const value = String(raw);
|
|
14362
|
+
return value.split(',').some(isToken) ? value : undefined;
|
|
14363
|
+
};
|
|
14364
|
+
const declared = valueIn(config.env);
|
|
14365
|
+
if (declared !== undefined)
|
|
14366
|
+
return { value: declared, source: 'env' };
|
|
14367
|
+
const inherited = config.inheritEnv ? valueIn(process.env) : undefined;
|
|
14368
|
+
return inherited !== undefined ? { value: inherited, source: 'inherited' } : undefined;
|
|
14369
|
+
}
|
|
13079
14370
|
/**
|
|
13080
14371
|
* Connect a new MCPL server at runtime. Refreshes the tool list and
|
|
13081
14372
|
* notifies the agent of newly available tools. Throws if the MCPL
|
|
@@ -13141,6 +14432,12 @@ export class AgentFramework {
|
|
|
13141
14432
|
if (!config) {
|
|
13142
14433
|
throw new Error(`MCPL server "${id}" is not configured`);
|
|
13143
14434
|
}
|
|
14435
|
+
// Every connect reads the awareness ledger: its awareness barrier
|
|
14436
|
+
// reconciles it, and a stdio child taking the protective default derives
|
|
14437
|
+
// that from it. Read it before disconnecting, so an unreadable ledger
|
|
14438
|
+
// refuses the restart and leaves the running child up, whatever the
|
|
14439
|
+
// server's baseline or transport. (No outbox configured: nothing to read.)
|
|
14440
|
+
this.getPlacedReactionBaseline();
|
|
13144
14441
|
await this.disconnectMcplServer(id);
|
|
13145
14442
|
await this.connectMcplServer(config);
|
|
13146
14443
|
}
|
|
@@ -13207,9 +14504,11 @@ export class AgentFramework {
|
|
|
13207
14504
|
connection.on('push-event', async (params, responder) => {
|
|
13208
14505
|
const answer = answerOnce(responder, 'push/event', connection.id);
|
|
13209
14506
|
try {
|
|
13210
|
-
|
|
13211
|
-
|
|
13212
|
-
|
|
14507
|
+
// Only when a generation is current: with none, the event is handled in
|
|
14508
|
+
// the turn it arrives, as it always was (the push coalescer's ordering of
|
|
14509
|
+
// a push against its retraction depends on that).
|
|
14510
|
+
if (this.discordAwarenessBarrier)
|
|
14511
|
+
await this.awaitDiscordAwarenessBarriers();
|
|
13213
14512
|
// Checked here, after the barrier and before anything records the
|
|
13214
14513
|
// push as accepted: a stopped host refuses it.
|
|
13215
14514
|
if (this.queue?.isClosed) {
|
|
@@ -13237,9 +14536,8 @@ export class AgentFramework {
|
|
|
13237
14536
|
if (this.inferenceRouter && responder) {
|
|
13238
14537
|
const answer = answerOnce(responder, 'inference/request', connection.id);
|
|
13239
14538
|
try {
|
|
13240
|
-
|
|
13241
|
-
|
|
13242
|
-
await barrier.promise;
|
|
14539
|
+
if (this.discordAwarenessBarrier)
|
|
14540
|
+
await this.awaitDiscordAwarenessBarriers();
|
|
13243
14541
|
// A stopping host starts no inference for a server: refused, like a
|
|
13244
14542
|
// push, before the router runs.
|
|
13245
14543
|
if (this.queue?.isClosed) {
|
|
@@ -13331,9 +14629,8 @@ export class AgentFramework {
|
|
|
13331
14629
|
connection.on('channels-incoming', async (params, responder) => {
|
|
13332
14630
|
const answer = answerOnce(responder, 'channels/incoming', connection.id);
|
|
13333
14631
|
try {
|
|
13334
|
-
|
|
13335
|
-
|
|
13336
|
-
await barrier.promise;
|
|
14632
|
+
if (this.discordAwarenessBarrier)
|
|
14633
|
+
await this.awaitDiscordAwarenessBarriers();
|
|
13337
14634
|
if (this.queue?.isClosed) {
|
|
13338
14635
|
this.refuseServerItem('channels/incoming', connection.id, answer, { kind: 'refused' });
|
|
13339
14636
|
return;
|
|
@@ -13350,9 +14647,8 @@ export class AgentFramework {
|
|
|
13350
14647
|
return;
|
|
13351
14648
|
const answer = answerOnce(responder, 'host/command', connection.id);
|
|
13352
14649
|
try {
|
|
13353
|
-
|
|
13354
|
-
|
|
13355
|
-
await barrier.promise;
|
|
14650
|
+
if (this.discordAwarenessBarrier)
|
|
14651
|
+
await this.awaitDiscordAwarenessBarriers();
|
|
13356
14652
|
// A stopping host runs no host command: refused like the requests
|
|
13357
14653
|
// above, before its handler runs.
|
|
13358
14654
|
if (this.queue?.isClosed) {
|
|
@@ -13394,7 +14690,7 @@ export class AgentFramework {
|
|
|
13394
14690
|
// grant) before the data-plane barrier releases — otherwise the first
|
|
13395
14691
|
// post-reconnect events land while the grant is still empty and are
|
|
13396
14692
|
// rejected fail-closed instead of delivered.
|
|
13397
|
-
|
|
14693
|
+
const reregistered = (async () => {
|
|
13398
14694
|
try {
|
|
13399
14695
|
const config = this.mcplServerConfigs.get(connection.id);
|
|
13400
14696
|
if (config) {
|
|
@@ -13407,7 +14703,11 @@ export class AgentFramework {
|
|
|
13407
14703
|
this.releaseMcplDataPlaneGate(awarenessBarrier);
|
|
13408
14704
|
this.handleToolsListChanged(connection.id);
|
|
13409
14705
|
})();
|
|
13410
|
-
|
|
14706
|
+
// The gate completes, and the data planes go ready, only once both the
|
|
14707
|
+
// awareness drain and the re-registration have finished. A drain can
|
|
14708
|
+
// settle first (one already running, or little to send), and the grant
|
|
14709
|
+
// must exist before the first data-plane event flows.
|
|
14710
|
+
void Promise.all([awarenessBarrier.promise, reregistered]).then(() => {
|
|
13411
14711
|
if (awarenessBarrier.requiresBarrier
|
|
13412
14712
|
&& !this.completeMcplDataPlaneGate(awarenessBarrier))
|
|
13413
14713
|
return;
|
|
@@ -14466,9 +15766,21 @@ export class AgentFramework {
|
|
|
14466
15766
|
};
|
|
14467
15767
|
}
|
|
14468
15768
|
if (candidate.kind === 'reference') {
|
|
15769
|
+
// Point at fetch_reference only where it can deliver: not for a
|
|
15770
|
+
// reference this process holds no record of (fetch_reference
|
|
15771
|
+
// looks the id up in the same registry), nor, as the stub does,
|
|
15772
|
+
// for an unfetched one the fetcher would refuse as expired.
|
|
15773
|
+
const record = referenceRegistry.get(candidate.refId);
|
|
15774
|
+
const expiry = record && !record.fetchedPath ? referenceExpiry(record.testimony) : null;
|
|
14469
15775
|
throw new Error(`save_recent_image: image at index ${position} is a reference (${candidate.refId}, ${candidate.mediaType}, ` +
|
|
14470
15776
|
`from ${describeTool(candidate)}) — its bytes are not held here. Nothing written. ` +
|
|
14471
|
-
|
|
15777
|
+
(!record
|
|
15778
|
+
? 'This process holds no record of it (a restart or an eviction drops records, and a stub quoted ' +
|
|
15779
|
+
"from elsewhere never had one), so fetch_reference can't fetch it either. If its tool can produce " +
|
|
15780
|
+
'it again, re-run that tool for a fresh reference.'
|
|
15781
|
+
: expiry
|
|
15782
|
+
? `It can't be fetched either — ${EXPIRY_REASON[expiry]}.`
|
|
15783
|
+
: 'Fetch it with fetch_reference and save the materialized file instead.'));
|
|
14472
15784
|
}
|
|
14473
15785
|
if (candidate.ref === null) {
|
|
14474
15786
|
throw new Error(`save_recent_image: image at index ${position} is a tool-result image from ${describeTool(candidate)} ` +
|
|
@@ -14568,14 +15880,16 @@ export class AgentFramework {
|
|
|
14568
15880
|
consider({ kind: 'tool', ref: ph.ref, toolCallId, toolName, mediaType: ph.mediaType, messagesBack });
|
|
14569
15881
|
continue;
|
|
14570
15882
|
}
|
|
14571
|
-
// Reference stub: the registry's
|
|
14572
|
-
// still has the record)
|
|
14573
|
-
//
|
|
14574
|
-
//
|
|
14575
|
-
|
|
14576
|
-
|
|
15883
|
+
// Reference stub: the registry's claimed type (when this process
|
|
15884
|
+
// still has the record), else the first image/ token in the
|
|
15885
|
+
// stub's text, which can come from its name, says whether it was
|
|
15886
|
+
// an image. Either is read as the stub shows a type, sanitized
|
|
15887
|
+
// and bounded. Non-image references are not slots.
|
|
15888
|
+
const claimed = referenceRegistry.get(ph.refId)?.testimony.mimeType ?? ph.mediaType ?? undefined;
|
|
15889
|
+
const mediaType = referenceTypeLabel(claimed).toLowerCase();
|
|
15890
|
+
if (!mediaType.startsWith('image/'))
|
|
14577
15891
|
continue;
|
|
14578
|
-
consider({ kind: 'reference', refId: ph.refId, mediaType
|
|
15892
|
+
consider({ kind: 'reference', refId: ph.refId, mediaType, toolCallId, toolName, messagesBack });
|
|
14579
15893
|
}
|
|
14580
15894
|
};
|
|
14581
15895
|
// Same-batch results (e.g. a snapshot dispatched alongside the save)
|
|
@@ -14622,10 +15936,14 @@ export class AgentFramework {
|
|
|
14622
15936
|
const block = content[b];
|
|
14623
15937
|
if (block?.type === 'image') {
|
|
14624
15938
|
const inline = block.source?.type === 'base64' && typeof block.source.data === 'string';
|
|
15939
|
+
// The sender's word, shown in the receipt and stored with the
|
|
15940
|
+
// file, so it's taken as a label; with nothing left of it, it
|
|
15941
|
+
// counts as absent.
|
|
15942
|
+
const claimed = block.source?.mediaType;
|
|
14625
15943
|
consider({
|
|
14626
15944
|
kind: 'attachment',
|
|
14627
15945
|
data: inline ? block.source.data : null,
|
|
14628
|
-
mediaType:
|
|
15946
|
+
mediaType: (typeof claimed === 'string' ? sanitizeLabel(claimed, REFERENCE_LIMITS.mimeType) : '') || 'image/png',
|
|
14629
15947
|
messagesBack,
|
|
14630
15948
|
});
|
|
14631
15949
|
}
|