@animalabs/agent-framework 0.22.0 → 0.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -4
- package/dist/src/agent.d.ts.map +1 -1
- package/dist/src/agent.js +3 -1
- package/dist/src/agent.js.map +1 -1
- package/dist/src/api/server.d.ts.map +1 -1
- package/dist/src/api/server.js +41 -4
- package/dist/src/api/server.js.map +1 -1
- package/dist/src/framework.d.ts +319 -20
- package/dist/src/framework.d.ts.map +1 -1
- package/dist/src/framework.js +1500 -182
- package/dist/src/framework.js.map +1 -1
- package/dist/src/index.d.ts +2 -2
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/mcpl/index.d.ts +2 -0
- package/dist/src/mcpl/index.d.ts.map +1 -1
- package/dist/src/mcpl/index.js.map +1 -1
- package/dist/src/mcpl/reference-fetcher.d.ts +2 -1
- package/dist/src/mcpl/reference-fetcher.d.ts.map +1 -1
- package/dist/src/mcpl/reference-fetcher.js +20 -9
- package/dist/src/mcpl/reference-fetcher.js.map +1 -1
- package/dist/src/mcpl/references.d.ts +29 -6
- package/dist/src/mcpl/references.d.ts.map +1 -1
- package/dist/src/mcpl/references.js +57 -16
- package/dist/src/mcpl/references.js.map +1 -1
- package/dist/src/mcpl/server-connection.d.ts +28 -1
- package/dist/src/mcpl/server-connection.d.ts.map +1 -1
- package/dist/src/mcpl/server-connection.js +32 -4
- package/dist/src/mcpl/server-connection.js.map +1 -1
- package/dist/src/module-registry.d.ts +9 -3
- package/dist/src/module-registry.d.ts.map +1 -1
- package/dist/src/module-registry.js +3 -2
- package/dist/src/module-registry.js.map +1 -1
- package/dist/src/modules/workspace/index.d.ts.map +1 -1
- package/dist/src/modules/workspace/index.js +46 -4
- package/dist/src/modules/workspace/index.js.map +1 -1
- package/dist/src/modules/workspace/observe.d.ts +13 -3
- package/dist/src/modules/workspace/observe.d.ts.map +1 -1
- package/dist/src/modules/workspace/observe.js +24 -4
- package/dist/src/modules/workspace/observe.js.map +1 -1
- package/dist/src/modules/workspace/reconcile.d.ts +14 -0
- package/dist/src/modules/workspace/reconcile.d.ts.map +1 -1
- package/dist/src/modules/workspace/reconcile.js +10 -2
- package/dist/src/modules/workspace/reconcile.js.map +1 -1
- package/dist/src/operator-log.d.ts +41 -0
- package/dist/src/operator-log.d.ts.map +1 -1
- package/dist/src/operator-log.js.map +1 -1
- package/dist/src/provider-waits.d.ts +216 -0
- package/dist/src/provider-waits.d.ts.map +1 -0
- package/dist/src/provider-waits.js +482 -0
- package/dist/src/provider-waits.js.map +1 -0
- package/dist/src/refusal-reactions.d.ts +14 -12
- package/dist/src/refusal-reactions.d.ts.map +1 -1
- package/dist/src/refusal-reactions.js +14 -12
- package/dist/src/refusal-reactions.js.map +1 -1
- package/dist/src/tool-result-guard.d.ts +30 -13
- package/dist/src/tool-result-guard.d.ts.map +1 -1
- package/dist/src/tool-result-guard.js +71 -25
- package/dist/src/tool-result-guard.js.map +1 -1
- package/dist/src/types/events.d.ts +12 -0
- package/dist/src/types/events.d.ts.map +1 -1
- package/dist/src/types/module.d.ts +12 -4
- package/dist/src/types/module.d.ts.map +1 -1
- package/dist/src/types/trace.d.ts +14 -3
- package/dist/src/types/trace.d.ts.map +1 -1
- package/dist/test/discord-awareness-outbox.test.js +3 -0
- package/dist/test/discord-awareness-outbox.test.js.map +1 -1
- package/dist/test/explicit-prose-routing.test.js +0 -2
- package/dist/test/explicit-prose-routing.test.js.map +1 -1
- package/dist/test/framework.test.js +160 -1
- package/dist/test/framework.test.js.map +1 -1
- package/dist/test/helpers/coalescing-fixture.d.ts.map +1 -1
- package/dist/test/helpers/coalescing-fixture.js +3 -2
- package/dist/test/helpers/coalescing-fixture.js.map +1 -1
- package/dist/test/inference-exhaustion-rewind.test.js +142 -14
- package/dist/test/inference-exhaustion-rewind.test.js.map +1 -1
- package/dist/test/inference-failure-bounds.test.d.ts +2 -0
- package/dist/test/inference-failure-bounds.test.d.ts.map +1 -0
- package/dist/test/inference-failure-bounds.test.js +1201 -0
- package/dist/test/inference-failure-bounds.test.js.map +1 -0
- package/dist/test/late-events-after-stop.test.d.ts +2 -0
- package/dist/test/late-events-after-stop.test.d.ts.map +1 -0
- package/dist/test/late-events-after-stop.test.js +122 -0
- package/dist/test/late-events-after-stop.test.js.map +1 -0
- package/dist/test/mcpl-awareness-barrier.test.js +141 -0
- package/dist/test/mcpl-awareness-barrier.test.js.map +1 -1
- package/dist/test/mcpl-coalescing.test.js +78 -16
- package/dist/test/mcpl-coalescing.test.js.map +1 -1
- package/dist/test/mcpl-config-diagnostics.test.js +6 -3
- package/dist/test/mcpl-config-diagnostics.test.js.map +1 -1
- package/dist/test/mcpl-manifest-changed.test.js +2 -1
- package/dist/test/mcpl-manifest-changed.test.js.map +1 -1
- package/dist/test/mcpl-reference-fetcher.test.js +232 -4
- package/dist/test/mcpl-reference-fetcher.test.js.map +1 -1
- package/dist/test/mcpl-reference-fields.test.js +69 -2
- package/dist/test/mcpl-reference-fields.test.js.map +1 -1
- package/dist/test/mcpl-ws-transport.test.js +103 -0
- package/dist/test/mcpl-ws-transport.test.js.map +1 -1
- package/dist/test/ops-alert.test.js +2 -1
- package/dist/test/ops-alert.test.js.map +1 -1
- package/dist/test/placed-reaction-baseline.test.d.ts +2 -0
- package/dist/test/placed-reaction-baseline.test.d.ts.map +1 -0
- package/dist/test/placed-reaction-baseline.test.js +312 -0
- package/dist/test/placed-reaction-baseline.test.js.map +1 -0
- package/dist/test/provider-acceleration-cooldown.test.js +5 -1
- package/dist/test/provider-acceleration-cooldown.test.js.map +1 -1
- package/dist/test/provider-waits.test.d.ts +2 -0
- package/dist/test/provider-waits.test.d.ts.map +1 -0
- package/dist/test/provider-waits.test.js +2196 -0
- package/dist/test/provider-waits.test.js.map +1 -0
- package/dist/test/reaction-baseline-platform.test.d.ts +2 -0
- package/dist/test/reaction-baseline-platform.test.d.ts.map +1 -0
- package/dist/test/reaction-baseline-platform.test.js +124 -0
- package/dist/test/reaction-baseline-platform.test.js.map +1 -0
- package/dist/test/refusal-reaction-baseline.test.js +10 -3
- package/dist/test/refusal-reaction-baseline.test.js.map +1 -1
- package/dist/test/save-recent-image.test.js +138 -2
- package/dist/test/save-recent-image.test.js.map +1 -1
- package/dist/test/surgery-marks-receipt.test.d.ts +2 -0
- package/dist/test/surgery-marks-receipt.test.d.ts.map +1 -0
- package/dist/test/surgery-marks-receipt.test.js +540 -0
- package/dist/test/surgery-marks-receipt.test.js.map +1 -0
- package/dist/test/tool-result-guard.test.js +221 -7
- package/dist/test/tool-result-guard.test.js.map +1 -1
- package/dist/test/workspace-three-way.test.js +115 -0
- package/dist/test/workspace-three-way.test.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/agent.ts +4 -1
- package/src/api/server.ts +43 -4
- package/src/framework.ts +1538 -189
- package/src/index.ts +7 -2
- package/src/mcpl/index.ts +2 -0
- package/src/mcpl/reference-fetcher.ts +18 -8
- package/src/mcpl/references.ts +57 -13
- package/src/mcpl/server-connection.ts +55 -4
- package/src/module-registry.ts +17 -9
- package/src/modules/workspace/index.ts +44 -4
- package/src/modules/workspace/observe.ts +31 -5
- package/src/modules/workspace/reconcile.ts +23 -2
- package/src/operator-log.ts +30 -0
- package/src/provider-waits.ts +523 -0
- package/src/refusal-reactions.ts +14 -12
- package/src/tool-result-guard.ts +72 -24
- package/src/types/events.ts +12 -0
- package/src/types/module.ts +12 -4
- package/src/types/trace.ts +14 -3
package/src/framework.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ToolPresentation, presentationTools, isPresentationTool, renderCatalogue, type PresentationSnapshot } from "./tool-presentation.js";
|
|
2
2
|
import { dirname, join } from 'node:path';
|
|
3
|
-
import { INLINE_WITHHELD_TEXT, classifyBlock, isInlineContradiction, referenceRegistry, referenceStubOrNull } from './mcpl/references.js';
|
|
3
|
+
import { EXPIRY_REASON, INLINE_WITHHELD_TEXT, REFERENCE_LIMITS, classifyBlock, isInlineContradiction, isReferenceExpired, referenceExpiry, referenceRegistry, referenceStubOrNull, referenceTypeLabel, sanitizeLabel } from './mcpl/references.js';
|
|
4
4
|
import { ReferenceFetcher, DEFAULT_FETCH_MAX_BYTES, EAGER_FETCH_TIMEOUT_MS } from './mcpl/reference-fetcher.js';
|
|
5
5
|
import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
|
6
6
|
import { JsStore } from '@animalabs/chronicle';
|
|
@@ -62,7 +62,7 @@ import type {
|
|
|
62
62
|
} from './types/index.js';
|
|
63
63
|
import { ProcessQueueImpl } from './queue.js';
|
|
64
64
|
import { DEFAULT_SHUTDOWN_TIMEOUT_MS, validateShutdownTimeout, shutdownErrorMessage } from './shutdown.js';
|
|
65
|
-
import { REFUSAL_REACTIONS, REFUSAL_REACTION_FALLBACK } from './refusal-reactions.js';
|
|
65
|
+
import { REFUSAL_REACTIONS, REFUSAL_REACTION_FALLBACK, REFUSAL_REACTION_BASELINE } from './refusal-reactions.js';
|
|
66
66
|
import { Agent } from './agent.js';
|
|
67
67
|
import { ModuleRegistry, checkedToolResult, isStateExistsError, storableToolResult, thrownResult } from './module-registry.js';
|
|
68
68
|
import { McplServerRegistry } from './mcpl/server-registry.js';
|
|
@@ -88,6 +88,7 @@ import { InferenceRouter } from './mcpl/inference-router.js';
|
|
|
88
88
|
import { ChannelRegistry, type ChannelToolOrigin } from './mcpl/channel-registry.js';
|
|
89
89
|
import { ConversationRouter, CONVERSATION_ROUTING_DEPRECATION_NOTICE } from './mcpl/conversation-router.js';
|
|
90
90
|
import { safeSlice } from './safe-slice.js';
|
|
91
|
+
import { ProviderWaits, EVERY_MODEL, isReleaseModel, withoutControls, type ProviderWait } from './provider-waits.js';
|
|
91
92
|
import type { WorkspaceModule } from './modules/workspace/index.js';
|
|
92
93
|
import type { ToolResultGuard, WithheldSpill } from './tool-result-guard.js';
|
|
93
94
|
import {
|
|
@@ -394,6 +395,7 @@ import { EventGate, formatShadowWarning } from './gate/event-gate.js';
|
|
|
394
395
|
import { UsageTracker, type PersistedUsageState } from './usage/usage-tracker.js';
|
|
395
396
|
import type { SessionUsageSnapshot, UsageUpdatedEvent } from './usage/types.js';
|
|
396
397
|
import type { McplServerConnection } from './mcpl/server-connection.js';
|
|
398
|
+
import { isWebSocketTransport } from './mcpl/transport.js';
|
|
397
399
|
import type {
|
|
398
400
|
McplServerConfig,
|
|
399
401
|
McplHostCapabilities,
|
|
@@ -412,12 +414,14 @@ import {
|
|
|
412
414
|
DiscordAwarenessOutbox,
|
|
413
415
|
defaultDiscordAwarenessOutboxPath,
|
|
414
416
|
extractDiscordAwarenessRefs,
|
|
417
|
+
type DiscordAwarenessBatch,
|
|
415
418
|
} from './recovery/discord-awareness-outbox.js';
|
|
416
419
|
import {
|
|
417
420
|
OperatorActionError,
|
|
418
421
|
OperatorLog,
|
|
419
422
|
capIds,
|
|
420
423
|
defaultOperatorLogPath,
|
|
424
|
+
type SurgeryMarkerReceipt,
|
|
421
425
|
type OperatorLogEntry,
|
|
422
426
|
type OperatorLogInput,
|
|
423
427
|
type OperatorRequester,
|
|
@@ -608,6 +612,21 @@ interface DiscordAwarenessBarrier {
|
|
|
608
612
|
promise: Promise<DiscordAwarenessDrainOutcome>;
|
|
609
613
|
}
|
|
610
614
|
|
|
615
|
+
/** Several drains' outcomes as one: unavailable only if every one was. */
|
|
616
|
+
function combineDrainOutcomes(outcomes: DiscordAwarenessDrainOutcome[]): DiscordAwarenessDrainOutcome {
|
|
617
|
+
if (outcomes.every((outcome) => outcome.status === 'unavailable')) {
|
|
618
|
+
return {
|
|
619
|
+
status: 'unavailable',
|
|
620
|
+
accounted: outcomes.reduce((count, outcome) => count + (outcome.status === 'unavailable' ? outcome.accounted : 0), 0),
|
|
621
|
+
};
|
|
622
|
+
}
|
|
623
|
+
return {
|
|
624
|
+
status: 'delivered',
|
|
625
|
+
delivered: outcomes.reduce((count, outcome) => count + (outcome.status === 'delivered' ? outcome.delivered : 0), 0),
|
|
626
|
+
failed: outcomes.reduce((count, outcome) => count + (outcome.status === 'delivered' ? outcome.failed : 0), 0),
|
|
627
|
+
};
|
|
628
|
+
}
|
|
629
|
+
|
|
611
630
|
function normalizeDiscordAwarenessDeadline(value: number | undefined): number {
|
|
612
631
|
if (value === undefined || !Number.isFinite(value) || value <= 0) {
|
|
613
632
|
return DEFAULT_DISCORD_AWARENESS_DEADLINE_MS;
|
|
@@ -814,11 +833,35 @@ function bodyGroupRun(
|
|
|
814
833
|
};
|
|
815
834
|
}
|
|
816
835
|
|
|
836
|
+
function describeMarkers(m: SurgeryMarkerReceipt): string {
|
|
837
|
+
if (m.status === 'queued') return `queued:${m.queued}`;
|
|
838
|
+
if (m.status === 'not-scheduled') return 'not-scheduled(batch retired)';
|
|
839
|
+
if (m.status === 'unresolved') return `unresolved(batch ${m.batchId} may still be delivered)`;
|
|
840
|
+
return 'none';
|
|
841
|
+
}
|
|
842
|
+
|
|
817
843
|
function describeRequester(r: OperatorRequester | undefined): string {
|
|
818
844
|
if (!r) return 'unknown';
|
|
819
845
|
return `${r.name ?? r.id ?? 'unknown'} (${r.via})`;
|
|
820
846
|
}
|
|
821
847
|
|
|
848
|
+
/** What a held call or turn says of the wait that holds it: no call was
|
|
849
|
+
* made. The refusal of an auxiliary call and the trace of a turn held once
|
|
850
|
+
* compiled say it the same way. */
|
|
851
|
+
function providerWaitText(agentName: string, model: string, wait: ProviderWait): string {
|
|
852
|
+
return `Provider wait: ${model} is held for ${agentName} ` +
|
|
853
|
+
`${wait.until === null ? 'until an operator releases it' : `until ${new Date(wait.until).toISOString()}`} ` +
|
|
854
|
+
`(recorded ${new Date(wait.setAt).toISOString()}: ${wait.reason}); no call was made`;
|
|
855
|
+
}
|
|
856
|
+
|
|
857
|
+
/** Who asked for an operator act, as a durable record and a log line carry
|
|
858
|
+
* it. The name comes from a caller, so a control character (a forged log
|
|
859
|
+
* line) becomes a space, and it is bounded. */
|
|
860
|
+
function operatorLabel(by: string): string {
|
|
861
|
+
const label = safeSlice(withoutControls(by).trim(), 0, 120);
|
|
862
|
+
return label === '' ? 'operator' : label;
|
|
863
|
+
}
|
|
864
|
+
|
|
822
865
|
function hostCommandRequester(serverId: string, params: HostCommandParams): OperatorRequester {
|
|
823
866
|
return {
|
|
824
867
|
via: `host-command:${serverId}`,
|
|
@@ -848,11 +891,11 @@ class DefaultErrorPolicy implements ErrorPolicy {
|
|
|
848
891
|
maxRetries = 3;
|
|
849
892
|
|
|
850
893
|
onInferenceError(error: Error, _agentName: string, attempt: number): ErrorAction {
|
|
851
|
-
if (error
|
|
894
|
+
if (isMembraneError(error) && error.retryable === false) {
|
|
852
895
|
return { retry: false };
|
|
853
896
|
}
|
|
854
897
|
if (attempt < this.maxRetries) {
|
|
855
|
-
const delayMs = error
|
|
898
|
+
const delayMs = isMembraneError(error) && typeof error.retryAfterMs === 'number'
|
|
856
899
|
? error.retryAfterMs
|
|
857
900
|
: Math.pow(2, attempt) * 1000;
|
|
858
901
|
return { retry: true, delayMs };
|
|
@@ -894,6 +937,17 @@ interface ProviderAccelerationCooldown {
|
|
|
894
937
|
/** Armed by an auxiliary (compression) failure with no primary involved:
|
|
895
938
|
* release reopens auxiliary admission without synthesising an inference. */
|
|
896
939
|
auxiliaryOrigin?: boolean;
|
|
940
|
+
/** Set when this hold is a provider wait (a stated retry-after) for the
|
|
941
|
+
* agent's primary model: it holds primary turns only. Auxiliary calls are
|
|
942
|
+
* admitted per model by the provider waits themselves, so a compression
|
|
943
|
+
* model the provider did not limit keeps working. */
|
|
944
|
+
waitModel?: string;
|
|
945
|
+
/** Set on an acceleration hold whose length is the provider's stated wait
|
|
946
|
+
* (recorded for `model`), which replaced AF's own pacing for the failure.
|
|
947
|
+
* `ownUntil` is when that pacing alone would have ended the hold: a
|
|
948
|
+
* release of the stated wait shortens the hold to it, never past it. A
|
|
949
|
+
* host hold never carries this: its length is the host's. */
|
|
950
|
+
statedWait?: { model: string; ownUntil: number };
|
|
897
951
|
}
|
|
898
952
|
interface ProviderAccelerationRecovery {
|
|
899
953
|
startedAt: number;
|
|
@@ -915,10 +969,33 @@ interface ProviderAccelerationReceipt {
|
|
|
915
969
|
stopReason: string;
|
|
916
970
|
}
|
|
917
971
|
const PROVIDER_ACCELERATION_DEFAULT_COOLDOWN_MS = 65_000;
|
|
972
|
+
/** Cap on waits the framework chooses itself. A provider's stated wait is
|
|
973
|
+
* never capped: it is a lower bound (see ProviderWaits). It is also the
|
|
974
|
+
* longest stated wait an ephemeral run sleeps in its turn: a longer one ends
|
|
975
|
+
* the run instead (nonOwningProviderWait). */
|
|
918
976
|
const PROVIDER_ACCELERATION_MAX_COOLDOWN_MS = 10 * 60_000;
|
|
919
977
|
const PROVIDER_ACCELERATION_JITTER_MS = 5_000;
|
|
978
|
+
/** Longest delay one setTimeout can represent (2^31-1 ms, about 24.8 days);
|
|
979
|
+
* a longer timer fires almost at once, so longer waits re-arm in steps. */
|
|
980
|
+
const MAX_TIMER_DELAY_MS = 2_147_483_647;
|
|
981
|
+
/** A stated wait recorded to hold a model longer than this, or until
|
|
982
|
+
* released, raises an ops alert. It is honoured in full either way: a hold
|
|
983
|
+
* that long is a quota-scale event or a provider's mistake, and a person
|
|
984
|
+
* decides which (release-provider-wait). Shorter waits are pacing. */
|
|
985
|
+
const PROVIDER_WAIT_ALERT_MS = 60 * 60_000;
|
|
986
|
+
/**
|
|
987
|
+
* A MembraneError, from this framework's copy of membrane or from another: the
|
|
988
|
+
* host supplies the Membrane instance (config.membrane), so with two copies
|
|
989
|
+
* installed its errors are instances of a class this framework's `instanceof`
|
|
990
|
+
* doesn't match. Recognised by name as well, as context-manager's errors are;
|
|
991
|
+
* its fields are the same, so callers still check each one's type.
|
|
992
|
+
*/
|
|
993
|
+
function isMembraneError(error: unknown): error is MembraneError {
|
|
994
|
+
return error instanceof MembraneError || (error instanceof Error && error.name === 'MembraneError');
|
|
995
|
+
}
|
|
996
|
+
|
|
920
997
|
function isOrganizationAccelerationRateLimit(error: Error): error is MembraneError {
|
|
921
|
-
if (!(error
|
|
998
|
+
if (!isMembraneError(error) || error.type !== 'rate_limit') return false;
|
|
922
999
|
return (
|
|
923
1000
|
/organization(?:'s)?[^\n]{0,120}maximum(?:\s+usage)?\s+increase\s+rate/i.test(error.message) ||
|
|
924
1001
|
/organization(?:'s)?[^\n]{0,120}acceleration(?:\s+limit)?/i.test(error.message)
|
|
@@ -993,6 +1070,9 @@ interface HostCommandParams {
|
|
|
993
1070
|
abandon?: boolean;
|
|
994
1071
|
/** For the `resume` command: override a failing feasibility verdict. */
|
|
995
1072
|
force?: boolean;
|
|
1073
|
+
/** For the `release-provider-wait` command: the model whose wait to release
|
|
1074
|
+
* (every model's when omitted). */
|
|
1075
|
+
model?: string;
|
|
996
1076
|
requesterId?: string;
|
|
997
1077
|
requesterName?: string;
|
|
998
1078
|
}
|
|
@@ -1015,6 +1095,14 @@ interface Deferred<T> {
|
|
|
1015
1095
|
reject: (reason?: unknown) => void;
|
|
1016
1096
|
}
|
|
1017
1097
|
|
|
1098
|
+
/** One onTrace subscription. Removing it, by its unsubscribe or by offTrace,
|
|
1099
|
+
* clears `active` as it splices it out, so a delivery already under way
|
|
1100
|
+
* skips it (see emitTrace). */
|
|
1101
|
+
interface TraceRegistration {
|
|
1102
|
+
listener: TraceEventListener;
|
|
1103
|
+
active: boolean;
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1018
1106
|
/** Per-run state for a framework-driven ephemeral agent: the settle signal
|
|
1019
1107
|
* the caller awaits plus the liveness/progress bookkeeping the watchdogs and
|
|
1020
1108
|
* the stream driver share. One map entry per run — created and torn down in
|
|
@@ -1043,11 +1131,256 @@ interface EphemeralRun {
|
|
|
1043
1131
|
toolCallsCount: number;
|
|
1044
1132
|
}
|
|
1045
1133
|
|
|
1046
|
-
/**
|
|
1047
|
-
*
|
|
1048
|
-
*
|
|
1049
|
-
|
|
1050
|
-
|
|
1134
|
+
/**
|
|
1135
|
+
* Longest failure text the inference-failure pipeline writes to one sink: the
|
|
1136
|
+
* reason in the exhaustion stderr line, a field of a logs/failures.log record,
|
|
1137
|
+
* an ops alert's message, the inference log's `error`, and a trace event's
|
|
1138
|
+
* `error` and `stack`. A failure's text is whatever its source put in
|
|
1139
|
+
* `Error.message`, and a provider can echo the entire rejected request there
|
|
1140
|
+
* (one OpenAI-compatible 400 echoed ~1.7 MB). The sinks with shorter bounds
|
|
1141
|
+
* name them below, and every cut goes through boundFailureText. This bound
|
|
1142
|
+
* doesn't depend on membrane bounding its own errors.
|
|
1143
|
+
*/
|
|
1144
|
+
const FAILURE_TEXT_MAX_CHARS = 2_000;
|
|
1145
|
+
/** Longest failure reason written into an agent's own context (the
|
|
1146
|
+
* `[inference-failed]` marker): enough for a provider's explanation, not a
|
|
1147
|
+
* self-dump the agent then carries in every later request. */
|
|
1148
|
+
const FAILURE_MARKER_REASON_MAX_CHARS = 600;
|
|
1149
|
+
/** Longest failure reason in the rewind marker, also the agent's own context:
|
|
1150
|
+
* enough to identify the failure class without a wall of provider JSON. */
|
|
1151
|
+
const REWIND_MARKER_REASON_MAX_CHARS = 160;
|
|
1152
|
+
/** Longest failure reason in an agent's status (`lastInference.lastError`) and
|
|
1153
|
+
* in the compile-refused ops alert. */
|
|
1154
|
+
const FAILURE_SUMMARY_MAX_CHARS = 300;
|
|
1155
|
+
/** Longest alert message posted to the ops webhook. */
|
|
1156
|
+
const OPS_WEBHOOK_MESSAGE_MAX_CHARS = 500;
|
|
1157
|
+
/** Longest provider error code recorded verbatim; real codes are short tokens.
|
|
1158
|
+
* A failure's name and type are bounded the same way. */
|
|
1159
|
+
const PROVIDER_ERROR_CODE_MAX_CHARS = 128;
|
|
1160
|
+
|
|
1161
|
+
const isHighSurrogate = (text: string, index: number): boolean => {
|
|
1162
|
+
const code = text.charCodeAt(index);
|
|
1163
|
+
return code >= 0xd800 && code <= 0xdbff;
|
|
1164
|
+
};
|
|
1165
|
+
const isLowSurrogate = (text: string, index: number): boolean => {
|
|
1166
|
+
const code = text.charCodeAt(index);
|
|
1167
|
+
return code >= 0xdc00 && code <= 0xdfff;
|
|
1168
|
+
};
|
|
1169
|
+
/** Whether a cut before `index` would separate the halves of a surrogate pair. */
|
|
1170
|
+
const splitsPair = (text: string, index: number): boolean =>
|
|
1171
|
+
index > 0 && index < text.length && isHighSurrogate(text, index - 1) && isLowSurrogate(text, index);
|
|
1172
|
+
|
|
1173
|
+
/**
|
|
1174
|
+
* `text` when it fits within `max` characters; otherwise its head and tail
|
|
1175
|
+
* around a marker stating how many of how many characters were omitted, the
|
|
1176
|
+
* whole within `max`. A cut that would separate a surrogate pair moves toward
|
|
1177
|
+
* omission, so the result never holds half of a pair the text held whole, and
|
|
1178
|
+
* bounding bounded text returns it as is. Each sink cuts the whole text, so
|
|
1179
|
+
* its marker states the whole length.
|
|
1180
|
+
*/
|
|
1181
|
+
function boundFailureText(text: string, max: number = FAILURE_TEXT_MAX_CHARS): string {
|
|
1182
|
+
if (text.length <= max) return text;
|
|
1183
|
+
const markerLength = ` …[${text.length} of ${text.length} characters omitted]… `.length;
|
|
1184
|
+
const budget = Math.max(0, max - markerLength);
|
|
1185
|
+
const tailLength = Math.floor(budget / 4);
|
|
1186
|
+
let headEnd = budget - tailLength;
|
|
1187
|
+
if (splitsPair(text, headEnd)) headEnd--;
|
|
1188
|
+
let tailStart = text.length - tailLength;
|
|
1189
|
+
if (splitsPair(text, tailStart)) tailStart++;
|
|
1190
|
+
return `${text.slice(0, headEnd)} …[${tailStart - headEnd} of ${text.length} characters omitted]… ${text.slice(tailStart)}`;
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
/** A provider error code, verbatim when token-sized; otherwise bounded. */
|
|
1194
|
+
function boundProviderErrorCode(code: unknown): string | undefined {
|
|
1195
|
+
if (typeof code !== 'string') return undefined;
|
|
1196
|
+
return boundFailureText(code, PROVIDER_ERROR_CODE_MAX_CHARS);
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
/**
|
|
1200
|
+
* A value's JSON for a failure sink, with every Error inside it written as its
|
|
1201
|
+
* whole projection (failureDescription): serialising an Error writes its own
|
|
1202
|
+
* fields, a MembraneError's whole `rawRequest` among them. The projection is
|
|
1203
|
+
* left uncut, as a nested string is, so the sink's one cut of the JSON states
|
|
1204
|
+
* its whole length. An Error is recognised by instanceof, so one from another
|
|
1205
|
+
* realm (a vm context) is written as its own JSON, as any object is. Throws,
|
|
1206
|
+
* or returns undefined, as JSON.stringify does.
|
|
1207
|
+
*/
|
|
1208
|
+
function failureJson(value: unknown): string | undefined {
|
|
1209
|
+
return JSON.stringify(value, (_key, nested: unknown) => (nested instanceof Error ? failureDescription(nested) : nested));
|
|
1210
|
+
}
|
|
1211
|
+
|
|
1212
|
+
/**
|
|
1213
|
+
* The whole text describeFailure cuts. For an Error, its head (name,
|
|
1214
|
+
* classification, status, provider code, each bounded) and its whole
|
|
1215
|
+
* message. For a primitive, its String. For another object, its String when
|
|
1216
|
+
* that's text of its own (a Buffer, an Error from another realm, a class
|
|
1217
|
+
* with its own toString). An array, or an object whose String is empty or
|
|
1218
|
+
* only its tag, such as `[object Object]`, is its JSON instead
|
|
1219
|
+
* (failureJson), so its fields and elements are kept; its tag stands when
|
|
1220
|
+
* JSON can't write it or finds nothing in it.
|
|
1221
|
+
*/
|
|
1222
|
+
function failureDescription(error: unknown): string {
|
|
1223
|
+
try {
|
|
1224
|
+
if (!(error instanceof Error)) {
|
|
1225
|
+
if (typeof error !== 'object' || error === null) return String(error);
|
|
1226
|
+
if (!Array.isArray(error)) {
|
|
1227
|
+
let text: string | undefined;
|
|
1228
|
+
try { text = String(error); } catch { /* no String of its own: its JSON, below */ }
|
|
1229
|
+
if (text && !/^\[object [^\]]*\]$/.test(text)) return text;
|
|
1230
|
+
}
|
|
1231
|
+
try {
|
|
1232
|
+
const json = failureJson(error);
|
|
1233
|
+
if (json !== undefined && json !== '{}') return json;
|
|
1234
|
+
} catch { /* its tag, below */ }
|
|
1235
|
+
return Object.prototype.toString.call(error);
|
|
1236
|
+
}
|
|
1237
|
+
const fields = error as Error & {
|
|
1238
|
+
type?: unknown; httpStatus?: unknown; providerErrorCode?: unknown; retryable?: unknown;
|
|
1239
|
+
};
|
|
1240
|
+
// Every component is remote or caller-supplied text, the name included.
|
|
1241
|
+
const classification = [
|
|
1242
|
+
typeof fields.type === 'string' ? boundFailureText(fields.type, PROVIDER_ERROR_CODE_MAX_CHARS) : undefined,
|
|
1243
|
+
typeof fields.httpStatus === 'number' ? `HTTP ${fields.httpStatus}` : undefined,
|
|
1244
|
+
boundProviderErrorCode(fields.providerErrorCode),
|
|
1245
|
+
typeof fields.retryable === 'boolean' ? `retryable=${fields.retryable}` : undefined,
|
|
1246
|
+
].filter((part): part is string => part !== undefined);
|
|
1247
|
+
const name = boundFailureText(String(error.name), PROVIDER_ERROR_CODE_MAX_CHARS);
|
|
1248
|
+
const head = classification.length > 0 ? `${name} (${classification.join(', ')})` : name;
|
|
1249
|
+
return `${head}: ${String(error.message)}`;
|
|
1250
|
+
} catch {
|
|
1251
|
+
return '[failure could not be described]';
|
|
1252
|
+
}
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
/**
|
|
1256
|
+
* A bounded string describing a failure, for logs that would otherwise print
|
|
1257
|
+
* the error object; it is one line unless one of its parts has newlines, as a
|
|
1258
|
+
* message can. Printing the object prints its message whole, in the stack,
|
|
1259
|
+
* and every own field: a MembraneError's `rawRequest` is the whole request
|
|
1260
|
+
* that failed, and Node's console prints each of its strings, a long system
|
|
1261
|
+
* prompt among them, up to 10,000 characters. The projection keeps what a
|
|
1262
|
+
* reader needs (name, classification, status, provider code, message) and
|
|
1263
|
+
* nothing else. The head's parts are bounded each, and the whole
|
|
1264
|
+
* (failureDescription) is cut once, so its marker states the length of the
|
|
1265
|
+
* head and the whole message.
|
|
1266
|
+
*/
|
|
1267
|
+
function describeFailure(error: unknown): string {
|
|
1268
|
+
return boundFailureText(failureDescription(error));
|
|
1269
|
+
}
|
|
1270
|
+
|
|
1271
|
+
/** A stack frame's line: an indent of spaces or tabs, then `at `. The indent
|
|
1272
|
+
* never crosses a line break, so a run of blank lines isn't rescanned from
|
|
1273
|
+
* each line's start. */
|
|
1274
|
+
const FRAME_LINE = /^[ \t]+at /m;
|
|
1275
|
+
|
|
1276
|
+
/**
|
|
1277
|
+
* describeFailure, then the error's stack frames, bounded, on the lines
|
|
1278
|
+
* below: for a log of a failure the framework's own code may have thrown,
|
|
1279
|
+
* where the frames say where. The stack opens with the message, and a
|
|
1280
|
+
* message's own lines can look like frames (an echoed request quoting a stack
|
|
1281
|
+
* trace), so when the message is the stack's header, before its first frame
|
|
1282
|
+
* line, the frames are the stack from its first frame line after the
|
|
1283
|
+
* message. Otherwise (a stack without its header, a message that isn't in
|
|
1284
|
+
* the stack or is empty) they are the stack from its first frame line. The
|
|
1285
|
+
* leak the projection closes was the object's own fields, not its stack.
|
|
1286
|
+
*/
|
|
1287
|
+
function describeFailureWithFrames(error: unknown): string {
|
|
1288
|
+
let frames: string | undefined;
|
|
1289
|
+
try {
|
|
1290
|
+
const stack = error instanceof Error ? error.stack : undefined;
|
|
1291
|
+
if (typeof stack === 'string') {
|
|
1292
|
+
const message = String((error as Error).message);
|
|
1293
|
+
// The message is the stack's header only before its first frame: a
|
|
1294
|
+
// stack without its header (copied, or frames only) can hold the
|
|
1295
|
+
// message's text inside a frame's name.
|
|
1296
|
+
const firstFrame = stack.search(FRAME_LINE);
|
|
1297
|
+
const at = message ? stack.indexOf(message) : -1;
|
|
1298
|
+
const frameLine = new RegExp(FRAME_LINE.source, 'gm');
|
|
1299
|
+
frameLine.lastIndex = at >= 0 && (firstFrame < 0 || at < firstFrame) ? at + message.length : 0;
|
|
1300
|
+
const found = frameLine.exec(stack);
|
|
1301
|
+
if (found) frames = boundFailureText(stack.slice(found.index));
|
|
1302
|
+
}
|
|
1303
|
+
} catch { /* the projection alone */ }
|
|
1304
|
+
const described = describeFailure(error);
|
|
1305
|
+
return frames === undefined ? described : `${described}\n${frames}`;
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
/**
|
|
1309
|
+
* A failure record's fields, each bounded, for logs/failures.log and the
|
|
1310
|
+
* ops:alert trace: a string to FAILURE_TEXT_MAX_CHARS; an Error to its
|
|
1311
|
+
* projection, since serialising one writes its own fields, a MembraneError's
|
|
1312
|
+
* whole `rawRequest` among them; any other value to its JSON, with each Error
|
|
1313
|
+
* inside it as its projection (failureJson), as the plain data that JSON
|
|
1314
|
+
* describes when it fits FAILURE_TEXT_MAX_CHARS and as the JSON text,
|
|
1315
|
+
* bounded, when it doesn't. So the trace's data is what failures.log writes:
|
|
1316
|
+
* a Map or an object with toJSON whose JSON is small never hands a listener
|
|
1317
|
+
* the value itself.
|
|
1318
|
+
*/
|
|
1319
|
+
function boundFailureFields(record: Record<string, unknown>): Record<string, unknown> {
|
|
1320
|
+
const bounded: Record<string, unknown> = {};
|
|
1321
|
+
for (const [key, value] of Object.entries(record)) {
|
|
1322
|
+
if (typeof value === 'string') bounded[key] = boundFailureText(value);
|
|
1323
|
+
else if (value instanceof Error) bounded[key] = describeFailure(value);
|
|
1324
|
+
else {
|
|
1325
|
+
let json: string | undefined;
|
|
1326
|
+
try { json = failureJson(value); } catch { bounded[key] = describeFailure(value); continue; }
|
|
1327
|
+
// A value JSON writes nothing for (a function, undefined) is left out,
|
|
1328
|
+
// as the log's JSON leaves it out.
|
|
1329
|
+
if (json === undefined) continue;
|
|
1330
|
+
bounded[key] = json.length <= FAILURE_TEXT_MAX_CHARS ? JSON.parse(json) : boundFailureText(json);
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
return bounded;
|
|
1334
|
+
}
|
|
1335
|
+
|
|
1336
|
+
/**
|
|
1337
|
+
* Whether the poison-history breaker may shed the newest exchange for a
|
|
1338
|
+
* failure: an `invalid_request` at HTTP 400, 413 or 422 or with no status, or
|
|
1339
|
+
* a `context_length` at 413. The status is a heuristic for when a shed may
|
|
1340
|
+
* help, not proof that the history is at fault.
|
|
1341
|
+
*
|
|
1342
|
+
* 400 and 422 are how a provider answers a request that fails its
|
|
1343
|
+
* validation, as a history it can't accept does, but other refusals come
|
|
1344
|
+
* back the same way, such as Anthropic's 400 for a credit balance that's too
|
|
1345
|
+
* low. A 413 is the request's size. When the newest exchange made it too
|
|
1346
|
+
* large, as an oversized attachment does, shedding that exchange may bring
|
|
1347
|
+
* the request back under the limit, and nothing else in the framework
|
|
1348
|
+
* recovers a 413; when the history as a whole is too large, the shed doesn't
|
|
1349
|
+
* help, and the breaker stops at its rewind cap. The 413 arm takes both
|
|
1350
|
+
* labels a 413 arrives under, invalid_request and context_length, because
|
|
1351
|
+
* membrane's label for a 413 differs by version and adapter. Other statuses
|
|
1352
|
+
* keep the history, since shed history can't be got back and the status
|
|
1353
|
+
* gives no sign that a shed would help: a 404 is a model or an endpoint that
|
|
1354
|
+
* doesn't exist, and 405, 409, 410 and 415 are about the HTTP request or the
|
|
1355
|
+
* resource, as a rule, rather than the conversation.
|
|
1356
|
+
*
|
|
1357
|
+
* The table test in test/inference-exhaustion-rewind.test.ts is this
|
|
1358
|
+
* predicate's contract. The notes below on which membrane version gives which
|
|
1359
|
+
* label are its history, and will go stale as hosts move to newer releases.
|
|
1360
|
+
*
|
|
1361
|
+
* Which failures reach the gate with which label depends on the membrane:
|
|
1362
|
+
* - As membrane 0.5.86 classifies them: Anthropic files a 400 as
|
|
1363
|
+
* invalid_request (after its context check) and its other 4xx as unknown.
|
|
1364
|
+
* The chat adapters never file one as invalid_request, since they classify
|
|
1365
|
+
* by the error's message text, and a context_length they file that way
|
|
1366
|
+
* carries status 400 whatever the response's was. The raw-fetch Responses
|
|
1367
|
+
* adapter files every status it doesn't list (404, 405, 409, 410, 413,
|
|
1368
|
+
* 415, 422 …) as invalid_request, so it alone produces an invalid_request
|
|
1369
|
+
* at 413 or 422, and no 0.5.86 adapter produces a context_length at 413.
|
|
1370
|
+
* - From membrane #56 (merged, not yet in a release), one table serves every
|
|
1371
|
+
* adapter. Auth codes and context overflow are checked on a 400 or 422
|
|
1372
|
+
* first, a 404 is invalid_request, a 413 is context_length, and other 4xx
|
|
1373
|
+
* are invalid_request. So a 413 still sheds, an Anthropic one included,
|
|
1374
|
+
* while a context_length at 400 or 422, a token overflow, keeps the
|
|
1375
|
+
* history. The 422 arm then also covers an OpenAI-compatible server, such
|
|
1376
|
+
* as a FastAPI one, that answers a validation failure with 422.
|
|
1377
|
+
*
|
|
1378
|
+
* An error with no status, from a membrane that reports none, keeps the rule
|
|
1379
|
+
* the breaker had before: the type alone.
|
|
1380
|
+
*/
|
|
1381
|
+
export function shedMayHelp(errorType: string | undefined, httpStatus: number | undefined): boolean {
|
|
1382
|
+
if (httpStatus === 413) return errorType === 'invalid_request' || errorType === 'context_length';
|
|
1383
|
+
return errorType === 'invalid_request' && (httpStatus === undefined || httpStatus === 400 || httpStatus === 422);
|
|
1051
1384
|
}
|
|
1052
1385
|
|
|
1053
1386
|
/** The most of a thrown value's or a tool error's text that reaches a log line or a server's error answer. */
|
|
@@ -1174,7 +1507,7 @@ export class AgentFramework {
|
|
|
1174
1507
|
private quiesced = false;
|
|
1175
1508
|
private quiesceReason?: string;
|
|
1176
1509
|
private quiescedAt?: number;
|
|
1177
|
-
private traceListeners:
|
|
1510
|
+
private traceListeners: TraceRegistration[] = [];
|
|
1178
1511
|
private syncIntervalMs: number;
|
|
1179
1512
|
private syncTimer: ReturnType<typeof setInterval> | null = null;
|
|
1180
1513
|
private maintenanceIntervalMs: number;
|
|
@@ -1190,6 +1523,10 @@ export class AgentFramework {
|
|
|
1190
1523
|
private providerAccelerationDefaultCooldownMs = PROVIDER_ACCELERATION_DEFAULT_COOLDOWN_MS;
|
|
1191
1524
|
private providerAccelerationJitterMs = PROVIDER_ACCELERATION_JITTER_MS;
|
|
1192
1525
|
private providerHoldHook: ProviderHoldHook | undefined;
|
|
1526
|
+
/** Provider waits (stated retry-after) per (agent, model), kept across restarts. */
|
|
1527
|
+
private providerWaits: ProviderWaits | undefined;
|
|
1528
|
+
/** Wakers for error-policy retry waits in progress: stop() ends them. */
|
|
1529
|
+
private retryWaitWakers = new Set<() => void>();
|
|
1193
1530
|
private providerAdmissionClosed = false;
|
|
1194
1531
|
/** Invalidates framework turns still assembling when stop begins. */
|
|
1195
1532
|
private lifecycleGeneration = 0;
|
|
@@ -1621,6 +1958,15 @@ export class AgentFramework {
|
|
|
1621
1958
|
this.operatorLog = operatorLog;
|
|
1622
1959
|
this.store = store;
|
|
1623
1960
|
this.ownsStore = ownsStore;
|
|
1961
|
+
// Unreadable recorded waits hold every agent's provider calls, and a
|
|
1962
|
+
// persistent cause never reads again: a person has to hear of it.
|
|
1963
|
+
this.providerWaits = new ProviderWaits(store, {
|
|
1964
|
+
onUnreadable: (error) => this.opsAlert('provider-wait', 'framework',
|
|
1965
|
+
`recorded provider waits could not be read (${error}): no provider call is admitted for any agent ` +
|
|
1966
|
+
'until they read again (re-read every 30 s) or an operator releases an agent\'s waits with ' +
|
|
1967
|
+
'release-provider-wait; such a release lasts for this process only',
|
|
1968
|
+
{ data: { model: EVERY_MODEL, until: null } }),
|
|
1969
|
+
});
|
|
1624
1970
|
this.membrane = membrane;
|
|
1625
1971
|
this.inferencePolicy = inferencePolicy;
|
|
1626
1972
|
this.errorPolicy = errorPolicy;
|
|
@@ -1638,14 +1984,16 @@ export class AgentFramework {
|
|
|
1638
1984
|
});
|
|
1639
1985
|
|
|
1640
1986
|
// Initialize module registry with callbacks
|
|
1641
|
-
|
|
1987
|
+
// A module's events, through pushEvent() or its deprecated queue handle,
|
|
1988
|
+
// go through pushModuleEvent(), which names the module.
|
|
1989
|
+
this.moduleRegistry = new ModuleRegistry(store, { push: (event, moduleName) => this.pushModuleEvent(event, moduleName, false) }, {
|
|
1642
1990
|
getAgents: () => Array.from(this.agents.values()),
|
|
1643
1991
|
addMessage: (p, c, m) => this.addMessage(p, c, m),
|
|
1644
1992
|
editMessage: (id, c) => this.editMessage(id, c),
|
|
1645
1993
|
removeMessage: (id) => this.removeMessage(id),
|
|
1646
1994
|
getMessage: (id) => this.getMessage(id),
|
|
1647
1995
|
queryMessages: (filter) => this.queryMessages(filter),
|
|
1648
|
-
pushEvent: (event) => this.
|
|
1996
|
+
pushEvent: (event, moduleName) => this.pushModuleEvent(event, moduleName, true),
|
|
1649
1997
|
onTrace: (listener) => this.onTrace(listener),
|
|
1650
1998
|
// The ONLY entry that confers module origin — a private closure the
|
|
1651
1999
|
// framework hands to its own module registry. The public
|
|
@@ -2179,6 +2527,7 @@ export class AgentFramework {
|
|
|
2179
2527
|
// store is still open, rather than leaving them to a reboot replay.
|
|
2180
2528
|
this.ackDeferredWrites();
|
|
2181
2529
|
this.providerAdmissionClosed = true;
|
|
2530
|
+
for (const wake of [...this.retryWaitWakers]) wake();
|
|
2182
2531
|
this.lifecycleGeneration++;
|
|
2183
2532
|
// The generation this stop ends; withheld spills close for it alone, so a
|
|
2184
2533
|
// start() made before this stop finishes reopens them for its own.
|
|
@@ -2340,7 +2689,13 @@ export class AgentFramework {
|
|
|
2340
2689
|
private providerGateBlocked(agentName: string): boolean {
|
|
2341
2690
|
const gate = this.providerGates.get(agentName);
|
|
2342
2691
|
return (gate?.primaryDepth ?? 0) > 0 || (gate?.primaryPending ?? false) ||
|
|
2343
|
-
this.
|
|
2692
|
+
this.cooldownBlocksAuxiliary(agentName);
|
|
2693
|
+
}
|
|
2694
|
+
/** Acceleration and host holds park the whole agent; a provider wait holds
|
|
2695
|
+
* only its own model, which auxiliary admission checks per call. */
|
|
2696
|
+
private cooldownBlocksAuxiliary(agentName: string): boolean {
|
|
2697
|
+
const cooldown = this.providerAccelerationCooldowns.get(agentName);
|
|
2698
|
+
return cooldown !== undefined && cooldown.waitModel === undefined;
|
|
2344
2699
|
}
|
|
2345
2700
|
private acquirePrimaryProviderGate(agentName: string): void {
|
|
2346
2701
|
const gate = this.providerGate(agentName); gate.primaryPending = false; gate.primaryDepth++;
|
|
@@ -2351,7 +2706,7 @@ export class AgentFramework {
|
|
|
2351
2706
|
}
|
|
2352
2707
|
private flushAuxiliaryAdmission(agentName: string): void {
|
|
2353
2708
|
const gate = this.providerGate(agentName);
|
|
2354
|
-
if (gate.primaryDepth > 0 || gate.primaryPending || this.
|
|
2709
|
+
if (gate.primaryDepth > 0 || gate.primaryPending || this.cooldownBlocksAuxiliary(agentName)) return;
|
|
2355
2710
|
for (const resolve of gate.auxiliaryWaiters.splice(0)) resolve();
|
|
2356
2711
|
}
|
|
2357
2712
|
private async waitForAuxiliaryIdle(agentName: string): Promise<void> {
|
|
@@ -2359,8 +2714,9 @@ export class AgentFramework {
|
|
|
2359
2714
|
if (gate.auxiliaryInFlight === 0) return;
|
|
2360
2715
|
await new Promise<void>((resolve) => gate.idleWaiters.push(resolve));
|
|
2361
2716
|
}
|
|
2362
|
-
private async withAuxiliaryAdmission<T>(agentName: string, run: () => Promise<T
|
|
2717
|
+
private async withAuxiliaryAdmission<T>(agentName: string, run: () => Promise<T>, model?: string): Promise<T> {
|
|
2363
2718
|
const gate = this.providerGate(agentName);
|
|
2719
|
+
this.refuseHeldProviderModel(agentName, model);
|
|
2364
2720
|
if (this.providerGateBlocked(agentName)) {
|
|
2365
2721
|
gate.deferredAuxiliary++;
|
|
2366
2722
|
try {
|
|
@@ -2372,14 +2728,20 @@ export class AgentFramework {
|
|
|
2372
2728
|
if (this.providerAdmissionClosed) {
|
|
2373
2729
|
throw new Error(`Provider admission closed while stopping (${agentName})`);
|
|
2374
2730
|
}
|
|
2731
|
+
// A wait recorded while this call was parked binds it too.
|
|
2732
|
+
this.refuseHeldProviderModel(agentName, model);
|
|
2375
2733
|
gate.auxiliaryInFlight++;
|
|
2376
2734
|
try { return await run(); }
|
|
2377
2735
|
catch (error) {
|
|
2378
2736
|
// Compression can be the first call to find a spent quota (idle agent,
|
|
2379
|
-
// scheduled maintenance).
|
|
2380
|
-
//
|
|
2737
|
+
// scheduled maintenance). Record a stated wait for this call's model,
|
|
2738
|
+
// and arm the host hold so later passes wait at the gate instead of
|
|
2739
|
+
// retrying into it; this pass still fails to its caller.
|
|
2381
2740
|
const agent = this.agents.get(agentName);
|
|
2382
|
-
if (agent && error instanceof Error)
|
|
2741
|
+
if (agent && error instanceof Error) {
|
|
2742
|
+
if (model !== undefined) this.recordProviderWait(agentName, model, error);
|
|
2743
|
+
this.holdProviderAcceleration(agent, error, undefined, true);
|
|
2744
|
+
}
|
|
2383
2745
|
throw error;
|
|
2384
2746
|
}
|
|
2385
2747
|
finally {
|
|
@@ -2388,21 +2750,58 @@ export class AgentFramework {
|
|
|
2388
2750
|
}
|
|
2389
2751
|
}
|
|
2390
2752
|
private auxiliaryMembraneFor(agentName: string): Membrane {
|
|
2753
|
+
return this.membraneAdmitting((run, model) => this.withAuxiliaryAdmission(agentName, run, model));
|
|
2754
|
+
}
|
|
2755
|
+
/**
|
|
2756
|
+
* A conversation agent's auxiliary calls (its context manager's
|
|
2757
|
+
* compression). A conversation agent takes no provider gate: admission
|
|
2758
|
+
* belongs to the persistent resident, and a generation-unique fork must not
|
|
2759
|
+
* leave a gate behind. But its stated waits are recorded for (agent, model)
|
|
2760
|
+
* as a resident's are, so they bind its auxiliary calls too: a held model
|
|
2761
|
+
* is refused without a call, and an auxiliary call's own stated wait is
|
|
2762
|
+
* recorded for it. (An ephemeral run records nothing, its name being
|
|
2763
|
+
* single-use, so its context manager keeps the plain membrane.)
|
|
2764
|
+
*/
|
|
2765
|
+
private conversationAuxiliaryMembraneFor(agentName: string): Membrane {
|
|
2766
|
+
return this.membraneAdmitting(async (run, model) => {
|
|
2767
|
+
this.refuseHeldProviderModel(agentName, model);
|
|
2768
|
+
try { return await run(); }
|
|
2769
|
+
catch (error) {
|
|
2770
|
+
if (model !== undefined && error instanceof Error) this.recordProviderWait(agentName, model, error);
|
|
2771
|
+
throw error;
|
|
2772
|
+
}
|
|
2773
|
+
});
|
|
2774
|
+
}
|
|
2775
|
+
/** The framework's membrane, with each complete() call passed through
|
|
2776
|
+
* `admit` along with the model it names. */
|
|
2777
|
+
private membraneAdmitting(admit: (run: () => Promise<unknown>, model: string | undefined) => Promise<unknown>): Membrane {
|
|
2391
2778
|
const target = this.membrane as unknown as Record<PropertyKey, unknown>;
|
|
2392
2779
|
return new Proxy(target, { get: (obj, prop, receiver) => {
|
|
2393
2780
|
const value = Reflect.get(obj, prop, receiver);
|
|
2394
2781
|
if (prop === 'complete' && typeof value === 'function') {
|
|
2395
|
-
return (...args: unknown[]) =>
|
|
2396
|
-
|
|
2782
|
+
return (...args: unknown[]) => {
|
|
2783
|
+
const model = (args[0] as { config?: { model?: unknown } } | undefined)?.config?.model;
|
|
2784
|
+
return admit(() => Reflect.apply(value, this.membrane, args) as Promise<unknown>,
|
|
2785
|
+
typeof model === 'string' ? model : undefined);
|
|
2786
|
+
};
|
|
2397
2787
|
}
|
|
2398
2788
|
return typeof value === 'function' ? value.bind(this.membrane) : value;
|
|
2399
2789
|
}}) as unknown as Membrane;
|
|
2400
2790
|
}
|
|
2401
|
-
|
|
2402
|
-
|
|
2791
|
+
/** An acceleration hold's length: AF's own pacing (`ownMs`: its default
|
|
2792
|
+
* plus jitter, capped), or the provider's stated wait in its place
|
|
2793
|
+
* (`stated`). The stated wait is a lower bound: jitter only adds to it,
|
|
2794
|
+
* and the cap applies only to AF's own default. A negative stated wait is
|
|
2795
|
+
* a stated 0, as in ProviderWaits: the 1 s floor, plus jitter. */
|
|
2796
|
+
private accelerationCooldownMs(agentName: string, error: MembraneError): { delayMs: number; ownMs: number; stated: boolean } {
|
|
2403
2797
|
let hash = 0; for (const ch of agentName) hash = ((hash * 31) + ch.charCodeAt(0)) >>> 0;
|
|
2404
2798
|
const jitter = this.providerAccelerationJitterMs > 0 ? hash % (this.providerAccelerationJitterMs + 1) : 0;
|
|
2405
|
-
|
|
2799
|
+
const ownMs = Math.min(PROVIDER_ACCELERATION_MAX_COOLDOWN_MS, Math.max(1_000, this.providerAccelerationDefaultCooldownMs) + jitter);
|
|
2800
|
+
const stated = error.retryAfterMs;
|
|
2801
|
+
if (typeof stated === 'number' && Number.isFinite(stated)) {
|
|
2802
|
+
return { delayMs: Math.max(1_000, stated) + jitter, ownMs, stated: true };
|
|
2803
|
+
}
|
|
2804
|
+
return { delayMs: ownMs, ownMs, stated: false };
|
|
2406
2805
|
}
|
|
2407
2806
|
private sameInferenceRequest(a: InferenceRequest, b: InferenceRequest): boolean {
|
|
2408
2807
|
return a.agentName === b.agentName && a.reason === b.reason && a.source === b.source &&
|
|
@@ -2413,10 +2812,14 @@ export class AgentFramework {
|
|
|
2413
2812
|
try {
|
|
2414
2813
|
const hold = this.providerHoldHook(error, agentName, { model: this.agents.get(agentName)?.model });
|
|
2415
2814
|
if (!hold || !Number.isFinite(hold.holdMs) || hold.holdMs <= 0) return undefined;
|
|
2815
|
+
// The reason is printed, kept in the cooldown and shown in the status
|
|
2816
|
+
// snapshot (cooldownReason); without the host's own, it is the error's
|
|
2817
|
+
// message, which can echo the whole request.
|
|
2416
2818
|
return { holdMs: Math.min(PROVIDER_ACCELERATION_MAX_COOLDOWN_MS, Math.max(1_000, hold.holdMs)),
|
|
2417
|
-
reason: typeof hold.reason === 'string' && hold.reason ? hold.reason : error.message };
|
|
2819
|
+
reason: boundFailureText(typeof hold.reason === 'string' && hold.reason ? hold.reason : String(error.message)) };
|
|
2418
2820
|
} catch (err) {
|
|
2419
|
-
|
|
2821
|
+
// The hook is handed the model's error, and may throw it back.
|
|
2822
|
+
console.error(`[provider-cooldown] providerHold hook threw for ${agentName}; treating as no hold: ${describeFailureWithFrames(err)}`);
|
|
2420
2823
|
return undefined;
|
|
2421
2824
|
}
|
|
2422
2825
|
}
|
|
@@ -2428,7 +2831,7 @@ export class AgentFramework {
|
|
|
2428
2831
|
if (!hold) return false;
|
|
2429
2832
|
clearTimeout(cooldown.timer);
|
|
2430
2833
|
cooldown.until = Date.now() + hold.holdMs;
|
|
2431
|
-
cooldown.timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agentName), hold.holdMs);
|
|
2834
|
+
cooldown.timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agentName), Math.min(hold.holdMs, MAX_TIMER_DELAY_MS));
|
|
2432
2835
|
cooldown.timer.unref?.();
|
|
2433
2836
|
if (hold.reason !== cooldown.reason) {
|
|
2434
2837
|
cooldown.reason = hold.reason;
|
|
@@ -2438,21 +2841,39 @@ export class AgentFramework {
|
|
|
2438
2841
|
}
|
|
2439
2842
|
private holdProviderAcceleration(agent: Agent, error: Error, trigger?: InferenceRequest, auxiliary = false): boolean {
|
|
2440
2843
|
if (this.ephemeralRuns.has(agent.name) || this.conversationAgentHomes.has(agent.name)) return false;
|
|
2844
|
+
// A provider's stated wait binds later calls to the primary model, across
|
|
2845
|
+
// restarts. (An auxiliary call records its own model's wait at the gate.)
|
|
2846
|
+
const wait = auxiliary ? undefined : this.recordProviderWait(agent.name, agent.model, error);
|
|
2441
2847
|
// The host knows more than the message wording does: ask it first. An
|
|
2442
2848
|
// auxiliary failure arms host holds only — the built-in acceleration
|
|
2443
2849
|
// cooldown stays a primary-path mechanism.
|
|
2444
2850
|
const hostHold = this.consultProviderHold(error, agent.name);
|
|
2445
|
-
|
|
2446
|
-
|
|
2447
|
-
const
|
|
2851
|
+
// A failure the provider says may not be retried keeps its terminal
|
|
2852
|
+
// disposition, whatever its wording (a stated wait still binds after it).
|
|
2853
|
+
const acceleration = !hostHold && !auxiliary && (error as MembraneError).retryable !== false
|
|
2854
|
+
&& isOrganizationAccelerationRateLimit(error);
|
|
2855
|
+
// The stated wait binds later calls whatever the failure was. Retaining
|
|
2856
|
+
// and retrying THIS request is for a failure the provider says may be
|
|
2857
|
+
// retried: a non-retryable one keeps its terminal disposition (marker,
|
|
2858
|
+
// failure streak), and the wait applies to the calls that follow it.
|
|
2859
|
+
if (!acceleration && !hostHold) {
|
|
2860
|
+
return wait !== undefined && isMembraneError(error) && error.retryable === true
|
|
2861
|
+
&& this.holdForProviderWait(agent, wait, trigger ? [trigger] : [], true);
|
|
2862
|
+
}
|
|
2863
|
+
const now = Date.now();
|
|
2864
|
+
const pacing = hostHold ? undefined : this.accelerationCooldownMs(agent.name, error as MembraneError);
|
|
2865
|
+
const delayMs = hostHold ? hostHold.holdMs : pacing!.delayMs;
|
|
2448
2866
|
const existing = this.providerAccelerationCooldowns.get(agent.name);
|
|
2449
2867
|
const held = existing?.heldRequests ?? [];
|
|
2450
2868
|
if (trigger && !held.some((r) => this.sameInferenceRequest(r, trigger))) held.push(trigger);
|
|
2451
2869
|
if (existing) clearTimeout(existing.timer);
|
|
2452
|
-
const timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agent.name), delayMs); timer.unref?.();
|
|
2870
|
+
const timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agent.name), Math.min(delayMs, MAX_TIMER_DELAY_MS)); timer.unref?.();
|
|
2453
2871
|
this.providerAccelerationCooldowns.set(agent.name, { startedAt: existing?.startedAt ?? now,
|
|
2454
|
-
until: now + delayMs, timer, heldRequests: held, reason: hostHold?.reason ?? error.message,
|
|
2872
|
+
until: now + delayMs, timer, heldRequests: held, reason: hostHold?.reason ?? boundFailureText(String(error.message)),
|
|
2455
2873
|
failures: (existing?.failures ?? 0) + 1, ...(hostHold ? { hostHoldError: error } : {}),
|
|
2874
|
+
// The stated wait (recorded above for agent.model) set this hold's
|
|
2875
|
+
// length: an operator release of it returns the hold to AF's own pacing.
|
|
2876
|
+
...(pacing?.stated ? { statedWait: { model: agent.model, ownUntil: now + pacing.ownMs } } : {}),
|
|
2456
2877
|
...(auxiliary && (!existing || existing.auxiliaryOrigin) ? { auxiliaryOrigin: true } : {}) });
|
|
2457
2878
|
if (auxiliary && !existing) {
|
|
2458
2879
|
console.error(`[provider-cooldown] agent=${agent.name} host hold (${hostHold!.reason}) from an auxiliary call — ` +
|
|
@@ -2461,16 +2882,96 @@ export class AgentFramework {
|
|
|
2461
2882
|
}
|
|
2462
2883
|
const gate = this.providerGate(agent.name); gate.primaryPending = true;
|
|
2463
2884
|
this.providerAccelerationRecoveries.set(agent.name, { startedAt: existing?.startedAt ?? now,
|
|
2464
|
-
failures: (existing?.failures ?? 0) + 1, heldRequests: held.length, reason: error.message });
|
|
2885
|
+
failures: (existing?.failures ?? 0) + 1, heldRequests: held.length, reason: boundFailureText(String(error.message)) });
|
|
2465
2886
|
console.error(`[provider-cooldown] agent=${agent.name} ` +
|
|
2466
2887
|
`${hostHold ? `host hold (${hostHold.reason})` : 'organization acceleration 429'} — ` +
|
|
2467
2888
|
`holding primary/auxiliary for ${delayMs}ms; ${held.length} request(s) retained`);
|
|
2468
2889
|
return true;
|
|
2469
2890
|
}
|
|
2891
|
+
/**
|
|
2892
|
+
* A provider's stated wait on a turn that doesn't own provider admission:
|
|
2893
|
+
* a conversation agent's or an ephemeral run's. Left to the error policy,
|
|
2894
|
+
* which returns a stated wait as it is, a 429 stating two hours would hold
|
|
2895
|
+
* the turn asleep two hours, up to three times, with nothing recorded and
|
|
2896
|
+
* nothing raised (membrane #56 review, M3). So:
|
|
2897
|
+
* - A conversation agent's turns come through the scheduler as a resident's
|
|
2898
|
+
* do, so it treats the wait as a resident does: recorded for (agent,
|
|
2899
|
+
* model), alerted past PROVIDER_WAIT_ALERT_MS, and the turn held until the
|
|
2900
|
+
* wait passes or an operator releases it ('held'). A non-retryable failure
|
|
2901
|
+
* records the wait for the calls after it and keeps its own disposition.
|
|
2902
|
+
* - An ephemeral run's name is single-use, so a wait recorded for it would
|
|
2903
|
+
* bind nothing, and its caller is waiting on it. A retryable stated wait
|
|
2904
|
+
* up to PROVIDER_ACCELERATION_MAX_COOLDOWN_MS is left to the error policy,
|
|
2905
|
+
* slept and retried. A longer one, or one that can't be represented, ends
|
|
2906
|
+
* the run now ('fail'): it is never retried before the provider's time,
|
|
2907
|
+
* and the caller hears why at once.
|
|
2908
|
+
* Anything else is the error policy's.
|
|
2909
|
+
*/
|
|
2910
|
+
private nonOwningProviderWait(agent: Agent, error: Error, trigger?: InferenceRequest): 'held' | 'fail' | undefined {
|
|
2911
|
+
// From either copy of membrane (isMembraneError), a wait is stated only
|
|
2912
|
+
// as a number, as the default error policy reads it.
|
|
2913
|
+
if (!isMembraneError(error) || typeof error.retryAfterMs !== 'number') return undefined;
|
|
2914
|
+
if (this.conversationAgentHomes.has(agent.name)) {
|
|
2915
|
+
const wait = this.recordProviderWait(agent.name, agent.model, error);
|
|
2916
|
+
return error.retryable === true && wait !== undefined && this.holdForProviderWait(agent, wait, trigger ? [trigger] : [], true)
|
|
2917
|
+
? 'held'
|
|
2918
|
+
: undefined;
|
|
2919
|
+
}
|
|
2920
|
+
if (this.ephemeralRuns.has(agent.name) && error.retryable === true
|
|
2921
|
+
&& !(error.retryAfterMs <= PROVIDER_ACCELERATION_MAX_COOLDOWN_MS)) {
|
|
2922
|
+
return 'fail';
|
|
2923
|
+
}
|
|
2924
|
+
return undefined;
|
|
2925
|
+
}
|
|
2926
|
+
/** Why an ephemeral run ended instead of sleeping a provider's stated wait,
|
|
2927
|
+
* for its caller. */
|
|
2928
|
+
private statedWaitNotSlept(error: Error): string {
|
|
2929
|
+
const stated = isMembraneError(error) ? error.retryAfterMs : undefined;
|
|
2930
|
+
const cap = `${PROVIDER_ACCELERATION_MAX_COOLDOWN_MS / 1000} s`;
|
|
2931
|
+
if (typeof stated !== 'number' || Number.isNaN(stated)) {
|
|
2932
|
+
return `${error.message} (the provider's stated wait is not a number, and an ephemeral run sleeps only a ` +
|
|
2933
|
+
`stated wait of ${cap} or less, so it ends now)`;
|
|
2934
|
+
}
|
|
2935
|
+
const asked = Number.isFinite(stated) ? `${Math.ceil(stated / 1000)} s` : 'longer than can be represented';
|
|
2936
|
+
return `${error.message} (the provider asked to wait ${asked}; an ephemeral run doesn't sleep a stated wait ` +
|
|
2937
|
+
`longer than ${cap}, so it ends now)`;
|
|
2938
|
+
}
|
|
2939
|
+
/** When a hold's timer next fires: at its deadline, stepped past what one
|
|
2940
|
+
* timer can hold. A hold with no deadline (a wait held until released, or
|
|
2941
|
+
* the stand-in for unreadable records) asks again as often as unreadable
|
|
2942
|
+
* records are read again: records that read again end it, and nothing
|
|
2943
|
+
* announces that. */
|
|
2944
|
+
private providerHoldTimerMs(remaining: number): number {
|
|
2945
|
+
return Number.isFinite(remaining)
|
|
2946
|
+
? Math.min(remaining, MAX_TIMER_DELAY_MS)
|
|
2947
|
+
: this.providerWaits?.retryReadMs ?? MAX_TIMER_DELAY_MS;
|
|
2948
|
+
}
|
|
2470
2949
|
private releaseProviderAccelerationCooldown(agentName: string): void {
|
|
2471
2950
|
const cooldown = this.providerAccelerationCooldowns.get(agentName); if (!cooldown) return;
|
|
2951
|
+
// A wait hold lasts exactly as long as its wait binds: longer once the
|
|
2952
|
+
// wait is extended, while it is indefinite, and over once no wait binds
|
|
2953
|
+
// (passed, released, or the stand-in for unreadable records lifted because
|
|
2954
|
+
// they read again). A different primary model now is not bound by it.
|
|
2955
|
+
if (cooldown.waitModel !== undefined && this.agents.get(agentName)?.model === cooldown.waitModel) {
|
|
2956
|
+
const wait = this.providerWaits?.active(agentName, cooldown.waitModel);
|
|
2957
|
+
cooldown.until = wait ? (wait.until ?? Number.POSITIVE_INFINITY) : Date.now();
|
|
2958
|
+
}
|
|
2959
|
+
const remaining = cooldown.until - Date.now();
|
|
2960
|
+
if (remaining > 0 && (cooldown.waitModel === undefined || this.agents.get(agentName)?.model === cooldown.waitModel)) {
|
|
2961
|
+
clearTimeout(cooldown.timer);
|
|
2962
|
+
cooldown.timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agentName), this.providerHoldTimerMs(remaining));
|
|
2963
|
+
cooldown.timer.unref?.();
|
|
2964
|
+
return;
|
|
2965
|
+
}
|
|
2472
2966
|
if (this.extendHostProviderHold(agentName, cooldown)) return;
|
|
2473
2967
|
clearTimeout(cooldown.timer); this.providerAccelerationCooldowns.delete(agentName);
|
|
2968
|
+
if (cooldown.waitModel !== undefined) {
|
|
2969
|
+
const model = this.agents.get(agentName)?.model;
|
|
2970
|
+
console.error(`[provider-wait] agent=${agentName} model=${cooldown.waitModel} ` +
|
|
2971
|
+
(model !== undefined && model !== cooldown.waitModel
|
|
2972
|
+
? `no longer holds primary turns: the primary model is now ${model} (the wait stays recorded)`
|
|
2973
|
+
: `primary hold ended after ${Date.now() - cooldown.startedAt}ms`));
|
|
2974
|
+
}
|
|
2474
2975
|
if (cooldown.auxiliaryOrigin && cooldown.heldRequests.length === 0) {
|
|
2475
2976
|
// Nobody asked for an inference while parked: just reopen the gate.
|
|
2476
2977
|
console.error(`[provider-cooldown] agent=${agentName} auxiliary hold released after ${Date.now() - cooldown.startedAt}ms`);
|
|
@@ -2485,6 +2986,201 @@ export class AgentFramework {
|
|
|
2485
2986
|
console.error(`[provider-cooldown] agent=${agentName} released after ${Date.now() - cooldown.startedAt}ms — ` +
|
|
2486
2987
|
`fresh compile queued with ${requests.length} retained request(s)`);
|
|
2487
2988
|
}
|
|
2989
|
+
/**
|
|
2990
|
+
* Wait an error policy's retry delay, all of it: the delay can be a
|
|
2991
|
+
* provider's retry-after, and a single timer past 2^31-1 ms fires almost at
|
|
2992
|
+
* once. Steps through longer waits; stop() wakes it so a shutting-down
|
|
2993
|
+
* framework is never left with a wait parked for days.
|
|
2994
|
+
*/
|
|
2995
|
+
private async waitForRetry(ms: number): Promise<void> {
|
|
2996
|
+
// An infinite delay waits until stop(), like any wait too long to end; a
|
|
2997
|
+
// delay that isn't a number, or is 0 or less, waits none.
|
|
2998
|
+
let remaining = ms > 0 ? ms : 0;
|
|
2999
|
+
while (remaining > 0 && !this.providerAdmissionClosed) {
|
|
3000
|
+
const step = Math.min(remaining, MAX_TIMER_DELAY_MS);
|
|
3001
|
+
remaining -= step;
|
|
3002
|
+
await new Promise<void>((resolve) => {
|
|
3003
|
+
const wakers = this.retryWaitWakers;
|
|
3004
|
+
const done = (): void => { clearTimeout(timer); wakers.delete(done); resolve(); };
|
|
3005
|
+
const timer = setTimeout(done, step);
|
|
3006
|
+
wakers.add(done);
|
|
3007
|
+
});
|
|
3008
|
+
}
|
|
3009
|
+
}
|
|
3010
|
+
|
|
3011
|
+
/**
|
|
3012
|
+
* Record a provider's stated wait (a classified error carrying
|
|
3013
|
+
* `retryAfterMs`) for (agent, model). Returns the wait that binds now,
|
|
3014
|
+
* which is the longer of this one and any outstanding one. A wait that
|
|
3015
|
+
* newly binds past PROVIDER_WAIT_ALERT_MS, or until released, raises an
|
|
3016
|
+
* ops alert.
|
|
3017
|
+
*/
|
|
3018
|
+
private recordProviderWait(agentName: string, model: string, error: Error): ProviderWait | undefined {
|
|
3019
|
+
// From either copy of membrane: the host supplies the Membrane instance,
|
|
3020
|
+
// and a wait is stated only as a number, as the default policy reads it.
|
|
3021
|
+
if (!isMembraneError(error) || typeof error.retryAfterMs !== 'number' || !this.providerWaits) return undefined;
|
|
3022
|
+
// The provider's words, bounded; a line break in them must not start a
|
|
3023
|
+
// line that reads as the framework's own where the reason is logged.
|
|
3024
|
+
const reason = safeSlice(withoutControls(error.message), 0, 300);
|
|
3025
|
+
const { wait, changed } = this.providerWaits.set(agentName, model, error.retryAfterMs, reason);
|
|
3026
|
+
// A wait already over (0, or a negative "retry now") holds nothing to report.
|
|
3027
|
+
const remaining = wait.until === null ? Number.POSITIVE_INFINITY : wait.until - Date.now();
|
|
3028
|
+
if (changed && remaining > 0) {
|
|
3029
|
+
const until = wait.until === null
|
|
3030
|
+
? 'until an operator releases it (the stated wait cannot be represented as an instant)'
|
|
3031
|
+
: `until ${new Date(wait.until).toISOString()}`;
|
|
3032
|
+
const stated = `${error.type}${error.httpStatus !== undefined ? ` HTTP ${error.httpStatus}` : ''}, retry-after ${String(error.retryAfterMs)}ms`;
|
|
3033
|
+
console.error(`[provider-wait] agent=${agentName} model=${model} holds ${until}: ${stated}`);
|
|
3034
|
+
if (remaining > PROVIDER_WAIT_ALERT_MS) {
|
|
3035
|
+
this.opsAlert('provider-wait', agentName,
|
|
3036
|
+
`provider wait on ${model} holds ${until} (${stated}: ${reason}); ` +
|
|
3037
|
+
'if the provider did not mean that, release it with release-provider-wait',
|
|
3038
|
+
{ data: { model, until: wait.until === null ? null : new Date(wait.until).toISOString() } });
|
|
3039
|
+
}
|
|
3040
|
+
}
|
|
3041
|
+
return wait;
|
|
3042
|
+
}
|
|
3043
|
+
|
|
3044
|
+
/**
|
|
3045
|
+
* Hold this agent's primary turns for a provider wait on its model. The
|
|
3046
|
+
* requests are kept and released together, into one fresh compile, when
|
|
3047
|
+
* the wait ends: the same path an acceleration cooldown takes, without
|
|
3048
|
+
* parking auxiliary calls to other models.
|
|
3049
|
+
*/
|
|
3050
|
+
private holdForProviderWait(agent: Agent, wait: ProviderWait, requests: InferenceRequest[], failed = false): boolean {
|
|
3051
|
+
const now = Date.now();
|
|
3052
|
+
const until = wait.until ?? Number.POSITIVE_INFINITY;
|
|
3053
|
+
if (until <= now) return false;
|
|
3054
|
+
const existing = this.providerAccelerationCooldowns.get(agent.name);
|
|
3055
|
+
const held = existing?.heldRequests ?? [];
|
|
3056
|
+
for (const request of requests) {
|
|
3057
|
+
if (!held.some((r) => this.sameInferenceRequest(r, request))) held.push(request);
|
|
3058
|
+
}
|
|
3059
|
+
// An acceleration or host hold already parks the whole agent; its release
|
|
3060
|
+
// re-checks this wait at admission.
|
|
3061
|
+
if (existing && existing.waitModel === undefined) return true;
|
|
3062
|
+
if (existing) clearTimeout(existing.timer);
|
|
3063
|
+
const timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agent.name), this.providerHoldTimerMs(until - now));
|
|
3064
|
+
timer.unref?.();
|
|
3065
|
+
this.providerAccelerationCooldowns.set(agent.name, {
|
|
3066
|
+
startedAt: existing?.startedAt ?? now, until, timer, heldRequests: held,
|
|
3067
|
+
// A failed call counts; a turn arriving at a wait recorded earlier doesn't.
|
|
3068
|
+
reason: wait.reason, failures: (existing?.failures ?? 0) + (failed ? 1 : 0), waitModel: wait.model,
|
|
3069
|
+
});
|
|
3070
|
+
if (!existing) {
|
|
3071
|
+
console.error(`[provider-wait] agent=${agent.name} model=${wait.model} holding primary turns ` +
|
|
3072
|
+
`${wait.until === null ? 'until released' : `until ${new Date(wait.until).toISOString()}`}; ${held.length} request(s) retained`);
|
|
3073
|
+
}
|
|
3074
|
+
return true;
|
|
3075
|
+
}
|
|
3076
|
+
|
|
3077
|
+
/**
|
|
3078
|
+
* Refuse an auxiliary call to a model the provider asked this agent to wait
|
|
3079
|
+
* on, without calling it. The refusal is a retryable rate limit carrying
|
|
3080
|
+
* the remaining wait, so a caller that paces (context-manager's compression
|
|
3081
|
+
* lane, from its corrective round #138 on) waits it out instead of spending
|
|
3082
|
+
* a call. It is marked
|
|
3083
|
+
* `providerAdmission: 'deferred'`: no call was made, so a caller can keep
|
|
3084
|
+
* the wait without counting a failed call (room-225 #47927).
|
|
3085
|
+
*/
|
|
3086
|
+
private refuseHeldProviderModel(agentName: string, model: string | undefined): void {
|
|
3087
|
+
// A call that names no model can't be matched to a model's wait, but the
|
|
3088
|
+
// hold that stands for unreadable records binds every model, so it binds
|
|
3089
|
+
// that call too (EVERY_MODEL: only that hold, or a release of it, says so).
|
|
3090
|
+
const wait = this.providerWaits?.active(agentName, model ?? EVERY_MODEL);
|
|
3091
|
+
if (!wait) return;
|
|
3092
|
+
const now = Date.now();
|
|
3093
|
+
throw Object.assign(new MembraneError({
|
|
3094
|
+
type: 'rate_limit',
|
|
3095
|
+
retryable: true,
|
|
3096
|
+
...(wait.until !== null ? { retryAfterMs: Math.max(0, wait.until - now) } : {}),
|
|
3097
|
+
message: providerWaitText(agentName, model ?? EVERY_MODEL, wait),
|
|
3098
|
+
rawError: undefined,
|
|
3099
|
+
}), { providerAdmission: 'deferred' as const });
|
|
3100
|
+
}
|
|
3101
|
+
|
|
3102
|
+
/**
|
|
3103
|
+
* Release provider waits: one model's for an agent, or every model's. The
|
|
3104
|
+
* explicit operator override for a wait that should not be honoured (the
|
|
3105
|
+
* passing of time needs no call). Recorded, so a restart does not bring the
|
|
3106
|
+
* wait back; held primary turns are released into one fresh compile. An
|
|
3107
|
+
* acceleration hold the released wait lengthened keeps only AF's own pacing
|
|
3108
|
+
* for its failure; a host hold stays the host's.
|
|
3109
|
+
*/
|
|
3110
|
+
releaseProviderWait(agentName: string, model?: string, by = 'operator'): Array<{
|
|
3111
|
+
model: string; until: string | null; release: 'recorded' | 'pending' | 'in-process override';
|
|
3112
|
+
}> {
|
|
3113
|
+
// A release names one of this framework's agents, or the agent of a wait
|
|
3114
|
+
// the listing shows (a disposed conversation fork's wait stays recorded):
|
|
3115
|
+
// an operator can release anything the listing shows. Any other name (a
|
|
3116
|
+
// typo, or '*' and 'framework', which stand for the hold on unreadable
|
|
3117
|
+
// records in its listing and its alert) is refused before anything is
|
|
3118
|
+
// recorded: it would lift nothing and still read as a success. That hold
|
|
3119
|
+
// is lifted one agent at a time.
|
|
3120
|
+
if (!this.agents.has(agentName) && !this.providerWaits?.recorded(agentName)) {
|
|
3121
|
+
throw new Error(`Unknown agent: ${safeSlice(JSON.stringify(agentName), 0, 120)}. A release names one of this ` +
|
|
3122
|
+
"framework's agents or an agent with a recorded wait, and the hold on unreadable provider waits is lifted " +
|
|
3123
|
+
'one agent at a time');
|
|
3124
|
+
}
|
|
3125
|
+
by = operatorLabel(String(by));
|
|
3126
|
+
// '*' (EVERY_MODEL, as providerWaits lists the unreadable-records hold)
|
|
3127
|
+
// names every model, the same as omitting it, everywhere below.
|
|
3128
|
+
if (model === EVERY_MODEL) model = undefined;
|
|
3129
|
+
const released = this.providerWaits?.release(agentName, model, by) ?? [];
|
|
3130
|
+
for (const { wait, release } of released) {
|
|
3131
|
+
console.error(`[provider-wait] agent=${agentName} model=${wait.model} released by ${by} (${release})`);
|
|
3132
|
+
}
|
|
3133
|
+
// The agent's context strategy may pace its own provider calls by the
|
|
3134
|
+
// same stated wait (context-manager's compression lane, from its
|
|
3135
|
+
// corrective round #138 on; 0.14.0 has no releaseCompressionPause, so this
|
|
3136
|
+
// calls nothing there). The release is the operator's word on that wait
|
|
3137
|
+
// too, so it is offered to the lane with its scope even when no wait
|
|
3138
|
+
// binds here: the lane can hold a pause after
|
|
3139
|
+
// this framework's own wait is gone (the records read again, the wait
|
|
3140
|
+
// passed, or it was released in another process). The strategy's own
|
|
3141
|
+
// backoff stays, and the receipt still lists only waits released here.
|
|
3142
|
+
// Read inside the try: the release is already recorded, so a strategy
|
|
3143
|
+
// that throws must not keep the hold below from being released.
|
|
3144
|
+
try {
|
|
3145
|
+
const strategy = this.agents.get(agentName)?.getContextManager().getStrategy() as
|
|
3146
|
+
{ releaseCompressionPause?: (model?: string) => boolean } | undefined;
|
|
3147
|
+
if (strategy?.releaseCompressionPause?.(model)) {
|
|
3148
|
+
console.error(`[provider-wait] agent=${agentName} model=${model ?? '*'} compression lane's provider wait released`);
|
|
3149
|
+
}
|
|
3150
|
+
} catch (error) {
|
|
3151
|
+
console.error(`[provider-wait] agent=${agentName} could not offer the release to the compression lane: ${safeSlice(error instanceof Error ? error.message : String(error), 0, 300)}`);
|
|
3152
|
+
}
|
|
3153
|
+
const cooldown = this.providerAccelerationCooldowns.get(agentName);
|
|
3154
|
+
const covers = (held: string | undefined): boolean => held !== undefined && (model === undefined || held === model);
|
|
3155
|
+
if (cooldown && covers(cooldown.waitModel)) {
|
|
3156
|
+
// A wait hold asks its wait again: released, the hold ends.
|
|
3157
|
+
this.releaseProviderAccelerationCooldown(agentName);
|
|
3158
|
+
} else if (cooldown?.statedWait && covers(cooldown.statedWait.model)) {
|
|
3159
|
+
// An acceleration hold the stated wait lengthened keeps only AF's own
|
|
3160
|
+
// pacing for that failure, and is never lengthened by a release. A host
|
|
3161
|
+
// hold carries no stated wait: its length stays the host's.
|
|
3162
|
+
cooldown.until = Math.min(cooldown.until, cooldown.statedWait.ownUntil);
|
|
3163
|
+
console.error(`[provider-wait] agent=${agentName} model=${cooldown.statedWait.model} acceleration hold keeps AF's own pacing only, ` +
|
|
3164
|
+
(cooldown.until > Date.now() ? `until ${new Date(cooldown.until).toISOString()}` : 'which has passed'));
|
|
3165
|
+
delete cooldown.statedWait;
|
|
3166
|
+
this.releaseProviderAccelerationCooldown(agentName);
|
|
3167
|
+
}
|
|
3168
|
+
return released.map(({ wait, release }) => ({
|
|
3169
|
+
model: wait.model, until: wait.until === null ? null : new Date(wait.until).toISOString(), release,
|
|
3170
|
+
}));
|
|
3171
|
+
}
|
|
3172
|
+
|
|
3173
|
+
/** Provider waits binding now, for health and doctor tooling. */
|
|
3174
|
+
providerWaitSnapshot(agentName?: string): Array<{ agent: string; model: string; until: string | null; reason: string; setAt: string }> {
|
|
3175
|
+
return (this.providerWaits?.list(agentName) ?? []).map((wait) => ({
|
|
3176
|
+
agent: wait.agent,
|
|
3177
|
+
model: wait.model,
|
|
3178
|
+
until: wait.until === null ? null : new Date(wait.until).toISOString(),
|
|
3179
|
+
reason: wait.reason,
|
|
3180
|
+
setAt: new Date(wait.setAt).toISOString(),
|
|
3181
|
+
}));
|
|
3182
|
+
}
|
|
3183
|
+
|
|
2488
3184
|
private recordProviderAccelerationRecovery(agent: Agent, request: NormalizedRequest | undefined, stopReason: string): void {
|
|
2489
3185
|
const recovery = this.providerAccelerationRecoveries.get(agent.name); if (!recovery?.releasedAt) return;
|
|
2490
3186
|
this.providerAccelerationRecoveries.delete(agent.name);
|
|
@@ -2519,7 +3215,10 @@ export class AgentFramework {
|
|
|
2519
3215
|
private startQueuedMaintenance(): void {
|
|
2520
3216
|
if (this.maintenancePass || !this.running) return;
|
|
2521
3217
|
const pass = this.runQueuedMaintenance().catch((error) => {
|
|
2522
|
-
|
|
3218
|
+
// The alert cuts its own copies from the whole message, so the
|
|
3219
|
+
// webhook's states the whole length.
|
|
3220
|
+
const wholeReason = error instanceof Error ? String(error.message) : failureDescription(error);
|
|
3221
|
+
const reason = boundFailureText(wholeReason);
|
|
2523
3222
|
console.error(
|
|
2524
3223
|
'[context-maintenance] pass failed:',
|
|
2525
3224
|
reason,
|
|
@@ -2527,7 +3226,7 @@ export class AgentFramework {
|
|
|
2527
3226
|
this.opsAlert(
|
|
2528
3227
|
'context-maintenance-failed',
|
|
2529
3228
|
'framework',
|
|
2530
|
-
|
|
3229
|
+
wholeReason,
|
|
2531
3230
|
{ data: { scope: 'pass' } },
|
|
2532
3231
|
);
|
|
2533
3232
|
});
|
|
@@ -2597,12 +3296,13 @@ export class AgentFramework {
|
|
|
2597
3296
|
record.ticks++;
|
|
2598
3297
|
}
|
|
2599
3298
|
} catch (error) {
|
|
2600
|
-
|
|
3299
|
+
const wholeError = error instanceof Error ? String(error.message) : failureDescription(error);
|
|
3300
|
+
record.error = boundFailureText(wholeError);
|
|
2601
3301
|
console.error(`[context-maintenance] agent=${agent.name} failed:`, record.error);
|
|
2602
3302
|
this.opsAlert(
|
|
2603
3303
|
'context-maintenance-failed',
|
|
2604
3304
|
agent.name,
|
|
2605
|
-
|
|
3305
|
+
wholeError,
|
|
2606
3306
|
{
|
|
2607
3307
|
data: {
|
|
2608
3308
|
scope: 'agent',
|
|
@@ -2712,18 +3412,40 @@ export class AgentFramework {
|
|
|
2712
3412
|
this.emitTrace({ type: 'process:received', processEvent: event });
|
|
2713
3413
|
}
|
|
2714
3414
|
|
|
3415
|
+
/**
|
|
3416
|
+
* A module's event. stop() closes the queue before it stops modules, so a
|
|
3417
|
+
* module's event can arrive after that: from a listener that fires while
|
|
3418
|
+
* stop() runs, or from the module's own stop(). Nothing will process it,
|
|
3419
|
+
* and a throw there can go unhandled or fail stop(), so it's dropped with a
|
|
3420
|
+
* log line naming the module. Otherwise it's queued as pushEvent() queues
|
|
3421
|
+
* it, or, through the deprecated queue handle, without the trace.
|
|
3422
|
+
*/
|
|
3423
|
+
private pushModuleEvent(event: ProcessEvent, moduleName: string | undefined, traced: boolean): void {
|
|
3424
|
+
if (this.queue.isClosed) {
|
|
3425
|
+
// A tool result takes pushEvent()'s own drop, whose line names the
|
|
3426
|
+
// call, the tool and its outcome.
|
|
3427
|
+
if (event.type === 'tool-result') {
|
|
3428
|
+
this.pushEvent(event);
|
|
3429
|
+
return;
|
|
3430
|
+
}
|
|
3431
|
+
console.error(`[framework] a '${event.type}' event from module ${moduleName ?? '(unnamed)'} arrived after stop() began — dropped`);
|
|
3432
|
+
return;
|
|
3433
|
+
}
|
|
3434
|
+
if (traced) this.pushEvent(event);
|
|
3435
|
+
else this.queue.push(event);
|
|
3436
|
+
}
|
|
3437
|
+
|
|
2715
3438
|
/**
|
|
2716
3439
|
* Add a trace event listener for observability.
|
|
2717
3440
|
*/
|
|
2718
3441
|
onTrace(listener: TraceEventListener): () => void {
|
|
2719
|
-
|
|
3442
|
+
const registration: TraceRegistration = { listener, active: true };
|
|
3443
|
+
this.traceListeners.push(registration);
|
|
2720
3444
|
// Return an unsubscribe so callers with a bounded lifetime (e.g. modules
|
|
2721
3445
|
// that get torn down and recreated on session switch) don't leak
|
|
2722
3446
|
// listeners. Existing callers that ignore the return value are unaffected.
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
if (idx >= 0) this.traceListeners.splice(idx, 1);
|
|
2726
|
-
};
|
|
3447
|
+
// It removes this subscription, whatever else the same function holds.
|
|
3448
|
+
return () => this.removeTraceRegistration(registration);
|
|
2727
3449
|
}
|
|
2728
3450
|
|
|
2729
3451
|
/**
|
|
@@ -2737,13 +3459,18 @@ export class AgentFramework {
|
|
|
2737
3459
|
}
|
|
2738
3460
|
|
|
2739
3461
|
/**
|
|
2740
|
-
* Remove a trace event listener
|
|
3462
|
+
* Remove a trace event listener: its first subscription, if it holds more
|
|
3463
|
+
* than one.
|
|
2741
3464
|
*/
|
|
2742
3465
|
offTrace(listener: TraceEventListener): void {
|
|
2743
|
-
const
|
|
2744
|
-
if (
|
|
2745
|
-
|
|
2746
|
-
|
|
3466
|
+
const registration = this.traceListeners.find((r) => r.listener === listener);
|
|
3467
|
+
if (registration) this.removeTraceRegistration(registration);
|
|
3468
|
+
}
|
|
3469
|
+
|
|
3470
|
+
private removeTraceRegistration(registration: TraceRegistration): void {
|
|
3471
|
+
registration.active = false;
|
|
3472
|
+
const index = this.traceListeners.indexOf(registration);
|
|
3473
|
+
if (index >= 0) this.traceListeners.splice(index, 1);
|
|
2747
3474
|
}
|
|
2748
3475
|
|
|
2749
3476
|
/**
|
|
@@ -2891,14 +3618,27 @@ export class AgentFramework {
|
|
|
2891
3618
|
* says fetch_reference instead of a path. */
|
|
2892
3619
|
private async autoFetchReferences(content: unknown[] | undefined, serverId: string, eager: boolean): Promise<void> {
|
|
2893
3620
|
if (!content) return;
|
|
2894
|
-
|
|
2895
|
-
|
|
3621
|
+
// Register every block first, then judge each uri once, by its latest
|
|
3622
|
+
// record. The registry keys records by uri and evicts the one registered
|
|
3623
|
+
// least recently, so a uri this result mentions keeps its record through
|
|
3624
|
+
// the result's later blocks, short of more distinct references in one
|
|
3625
|
+
// result than the registry holds; past that, a uri can get a second
|
|
3626
|
+
// record here, hence the key by uri. A uri that any of this result's
|
|
3627
|
+
// blocks calls expired takes no slot, whichever block registered last. A
|
|
3628
|
+
// later result's fresh expiry still supersedes, through register().
|
|
3629
|
+
const byUri = new Map<string, { record: import('./mcpl/references.js').ReferenceRecord; expiredHere: boolean }>();
|
|
2896
3630
|
for (const block of content) {
|
|
2897
3631
|
const c = classifyBlock(block);
|
|
2898
3632
|
if (c.kind !== 'reference') continue;
|
|
2899
3633
|
const record = referenceRegistry.register(c.testimony, serverId);
|
|
2900
|
-
|
|
3634
|
+
const expiredHere = byUri.get(c.testimony.uri)?.expiredHere === true || isReferenceExpired(c.testimony);
|
|
3635
|
+
byUri.set(c.testimony.uri, { record, expiredHere });
|
|
2901
3636
|
}
|
|
3637
|
+
if (!eager) return;
|
|
3638
|
+
const fetcher = this.getReferenceFetcher();
|
|
3639
|
+
const eligible = [...byUri.values()]
|
|
3640
|
+
.filter(({ record, expiredHere }) => !expiredHere && fetcher.isEagerEligible(record))
|
|
3641
|
+
.map(({ record }) => record);
|
|
2902
3642
|
// Parallel with a count cap: N small fetches cost one round-trip of
|
|
2903
3643
|
// wall-clock, and anything past the cap stays lazy (stub says so).
|
|
2904
3644
|
const MAX_EAGER_PER_RESULT = 8;
|
|
@@ -4634,7 +5374,8 @@ export class AgentFramework {
|
|
|
4634
5374
|
if (!run) return;
|
|
4635
5375
|
run.lastActivity = Date.now();
|
|
4636
5376
|
if (outcome.stopReason === 'exhausted') {
|
|
4637
|
-
|
|
5377
|
+
// The caller may log or relay the message, which can echo the whole request.
|
|
5378
|
+
run.settle.reject(new Error(boundFailureText(outcome.error ?? 'Unknown error')));
|
|
4638
5379
|
return;
|
|
4639
5380
|
}
|
|
4640
5381
|
run.settle.resolve({
|
|
@@ -5205,7 +5946,8 @@ export class AgentFramework {
|
|
|
5205
5946
|
const parts = incoming.channelId.split(':');
|
|
5206
5947
|
if (parts[0] !== 'discord') return;
|
|
5207
5948
|
const channelId = parts[parts.length - 1];
|
|
5208
|
-
|
|
5949
|
+
// Own names only: an inherited one such as 'constructor' takes the fallback.
|
|
5950
|
+
const emoji = Object.hasOwn(REFUSAL_REACTIONS, category) ? REFUSAL_REACTIONS[category] : REFUSAL_REACTION_FALLBACK;
|
|
5209
5951
|
// Resolve the MCPL server that owns the locus channel and call
|
|
5210
5952
|
// tools/call directly on its connection (bare tool name — no prefix
|
|
5211
5953
|
// games), bypassing the agent event queue so no synthetic tool-result
|
|
@@ -5330,9 +6072,10 @@ export class AgentFramework {
|
|
|
5330
6072
|
? `the model refused on them (content filter: ${category}). Their content is ` +
|
|
5331
6073
|
`not reproduced here (so this note can't re-trigger the filter); the ` +
|
|
5332
6074
|
`originals remain in the raw record`
|
|
5333
|
-
: `the model API kept
|
|
5334
|
-
`
|
|
5335
|
-
`
|
|
6075
|
+
: `the model API kept refusing the requests that carried them (${category}), ` +
|
|
6076
|
+
`so that the conversation could go on. Their content is not reproduced ` +
|
|
6077
|
+
`here (so this note can't trigger the same refusal); the originals remain ` +
|
|
6078
|
+
`in the raw record`;
|
|
5336
6079
|
const text =
|
|
5337
6080
|
`[refusal-rewind] The ${count} most recent turn(s) were set aside because ` +
|
|
5338
6081
|
`${why}. You are clear to continue — if you ` +
|
|
@@ -5495,6 +6238,10 @@ export class AgentFramework {
|
|
|
5495
6238
|
*
|
|
5496
6239
|
* host-status — report the quiesce state (drained / parked wakes /
|
|
5497
6240
|
* running background scripts).
|
|
6241
|
+
*
|
|
6242
|
+
* release-provider-wait — end an agent's provider wait (a provider's
|
|
6243
|
+
* stated retry-after, kept across restarts) for one `model` or all, as an
|
|
6244
|
+
* explicit, recorded operator override.
|
|
5498
6245
|
*/
|
|
5499
6246
|
private async handleHostCommand(
|
|
5500
6247
|
serverId: string,
|
|
@@ -5514,6 +6261,9 @@ export class AgentFramework {
|
|
|
5514
6261
|
/** Discord addresses removed by message-granular undo. The durable outbox
|
|
5515
6262
|
* owns eventual delivery; this is also returned for immediate surfaces. */
|
|
5516
6263
|
removedRefs?: Array<{ serverId: string; channelId: string; messageId: string }>;
|
|
6264
|
+
/** For message-granular undo: whether awareness marks were scheduled
|
|
6265
|
+
* (not whether Discord has accepted them). */
|
|
6266
|
+
markers?: SurgeryMarkerReceipt;
|
|
5517
6267
|
hidden?: number;
|
|
5518
6268
|
/** For `hide`: the Discord (channelId, messageId) of each removed message
|
|
5519
6269
|
* that carried one — so the surface can mark them with a reaction. */
|
|
@@ -5529,6 +6279,8 @@ export class AgentFramework {
|
|
|
5529
6279
|
/** For `nudge`: agent state at queue time ('idle' = runs immediately,
|
|
5530
6280
|
* else it runs once the current turn settles). */
|
|
5531
6281
|
agentStatus?: string;
|
|
6282
|
+
/** For `release-provider-wait`: the waits released. */
|
|
6283
|
+
released?: Array<{ model: string; until: string | null }>;
|
|
5532
6284
|
}> {
|
|
5533
6285
|
if (
|
|
5534
6286
|
params.command !== 'undo' &&
|
|
@@ -5538,7 +6290,8 @@ export class AgentFramework {
|
|
|
5538
6290
|
params.command !== 'quiesce' &&
|
|
5539
6291
|
params.command !== 'resume' &&
|
|
5540
6292
|
params.command !== 'maintain' &&
|
|
5541
|
-
params.command !== 'host-status'
|
|
6293
|
+
params.command !== 'host-status' &&
|
|
6294
|
+
params.command !== 'release-provider-wait'
|
|
5542
6295
|
) {
|
|
5543
6296
|
return { ok: false, error: `Unknown host command: ${String(params.command)}` };
|
|
5544
6297
|
}
|
|
@@ -5575,6 +6328,30 @@ export class AgentFramework {
|
|
|
5575
6328
|
if (params.command === 'host-status') {
|
|
5576
6329
|
return { ok: true, hostMode: this.getHostModeStatus() };
|
|
5577
6330
|
}
|
|
6331
|
+
// A release names its agent, as host.releaseProviderWait does: the first
|
|
6332
|
+
// agent by default could lift the wrong agent's waits. The name may be an
|
|
6333
|
+
// agent the framework no longer has whose wait the listing still shows
|
|
6334
|
+
// (a disposed conversation fork); releaseProviderWait decides that.
|
|
6335
|
+
if (params.command === 'release-provider-wait') {
|
|
6336
|
+
if (typeof params.agentName !== 'string' || params.agentName === '') {
|
|
6337
|
+
return { ok: false, error: 'release-provider-wait: agentName must name the agent whose waits to release' };
|
|
6338
|
+
}
|
|
6339
|
+
// Only an omitted model, or '*', releases every model: a malformed one
|
|
6340
|
+
// is refused here, before anything changes, never widened.
|
|
6341
|
+
if (!isReleaseModel(params.model)) {
|
|
6342
|
+
return { ok: false, error: "release-provider-wait: model must be a model name, or '*' (or omitted) for every model" };
|
|
6343
|
+
}
|
|
6344
|
+
const by = operatorLabel(String(requester));
|
|
6345
|
+
let released: ReturnType<AgentFramework['releaseProviderWait']>;
|
|
6346
|
+
try {
|
|
6347
|
+
released = this.releaseProviderWait(params.agentName, params.model, by);
|
|
6348
|
+
} catch (error) {
|
|
6349
|
+
return { ok: false, error: error instanceof Error ? error.message : String(error) };
|
|
6350
|
+
}
|
|
6351
|
+
console.error(`[host-command] release-provider-wait agent=${params.agentName} model=${params.model ?? '*'} by=${by} ` +
|
|
6352
|
+
`(server=${serverId}): ${released.length} wait(s) lifted`);
|
|
6353
|
+
return { ok: true, released };
|
|
6354
|
+
}
|
|
5578
6355
|
|
|
5579
6356
|
const agentName = params.agentName ?? [...this.agents.keys()][0];
|
|
5580
6357
|
if (!agentName || !this.agents.has(agentName)) {
|
|
@@ -5757,6 +6534,7 @@ export class AgentFramework {
|
|
|
5757
6534
|
ok: true,
|
|
5758
6535
|
messagesRemoved: r.messagesRemoved,
|
|
5759
6536
|
removedRefs: r.removedRefs,
|
|
6537
|
+
markers: r.markers,
|
|
5760
6538
|
lastVisible: r.lastVisible,
|
|
5761
6539
|
};
|
|
5762
6540
|
} catch (error) {
|
|
@@ -5793,7 +6571,9 @@ export class AgentFramework {
|
|
|
5793
6571
|
return { ok: true, undone: 0, requested, lastVisible: null };
|
|
5794
6572
|
}
|
|
5795
6573
|
|
|
5796
|
-
|
|
6574
|
+
// The branch has moved: delivery of the marks it changes starts behind
|
|
6575
|
+
// the data-plane gate, and the command doesn't wait for Discord.
|
|
6576
|
+
this.startDiscordAwarenessDelivery('turn undo', agentName);
|
|
5797
6577
|
|
|
5798
6578
|
return { ok: true, undone, requested, lastVisible: await this.lastVisiblePreview(agentName) };
|
|
5799
6579
|
}
|
|
@@ -6388,6 +7168,12 @@ export class AgentFramework {
|
|
|
6388
7168
|
* Discord messages that left the live context get awareness markers via the
|
|
6389
7169
|
* durable outbox, exactly as message-granular `undo` does.
|
|
6390
7170
|
*
|
|
7171
|
+
* Returns once the switch has landed: the body is applied, with a
|
|
7172
|
+
* marker-scheduling receipt (`markers`). It does not wait for Discord to
|
|
7173
|
+
* accept the reactions: delivery is started, behind the MCPL data-plane
|
|
7174
|
+
* gate, and continues without the caller or the store reservation waiting
|
|
7175
|
+
* on it; its per-message outcomes stay in the outbox ledger.
|
|
7176
|
+
*
|
|
6391
7177
|
* Throws `OperatorActionError` (`agent-busy`, `unknown-message`, …) — the
|
|
6392
7178
|
* agent must be idle; nothing is queued.
|
|
6393
7179
|
*/
|
|
@@ -6410,6 +7196,8 @@ export class AgentFramework {
|
|
|
6410
7196
|
* the group's last shard so the sharded message stays whole. */
|
|
6411
7197
|
tailMessageId: string;
|
|
6412
7198
|
removedRefs: Array<{ serverId: string; channelId: string; messageId: string }>;
|
|
7199
|
+
/** Whether awareness marks were scheduled; never a delivery claim. */
|
|
7200
|
+
markers: SurgeryMarkerReceipt;
|
|
6413
7201
|
lastVisible: { participant?: string; role?: string; preview?: string } | null;
|
|
6414
7202
|
}> {
|
|
6415
7203
|
const logBase: OperatorLogInput = {
|
|
@@ -6447,9 +7235,12 @@ export class AgentFramework {
|
|
|
6447
7235
|
}
|
|
6448
7236
|
|
|
6449
7237
|
// Gate + reserve the whole store (see reserveStoreForSurgery); held
|
|
6450
|
-
// until the switch has landed
|
|
7238
|
+
// until the switch has landed and its marker scheduling has been
|
|
7239
|
+
// recorded or reported, or the switch has been rolled back. Remote
|
|
7240
|
+
// delivery is not waited on.
|
|
6451
7241
|
const release = this.reserveStoreForSurgery('roll back', agentName);
|
|
6452
7242
|
let branchName: string;
|
|
7243
|
+
let markers: SurgeryMarkerReceipt;
|
|
6453
7244
|
try {
|
|
6454
7245
|
// Prepare the external side effect before switching Chronicle. If the
|
|
6455
7246
|
// process dies after the switch but before activate(), startup
|
|
@@ -6480,8 +7271,14 @@ export class AgentFramework {
|
|
|
6480
7271
|
{ cause: error },
|
|
6481
7272
|
);
|
|
6482
7273
|
}
|
|
6483
|
-
|
|
6484
|
-
|
|
7274
|
+
// The body change has landed. Nothing after this point turns the
|
|
7275
|
+
// rollback into a failure: marks are an outbox outcome, reported
|
|
7276
|
+
// apart from the body (`markers`).
|
|
7277
|
+
markers = this.activateSurgeryMarkers(markerBatch, 'rollback', agentName);
|
|
7278
|
+
// Installs the MCPL data-plane gate synchronously, before the store is
|
|
7279
|
+
// released, and starts delivery without awaiting it (the first
|
|
7280
|
+
// reaction request may already be on the wire when this returns).
|
|
7281
|
+
this.startDiscordAwarenessDelivery('rollback', agentName);
|
|
6485
7282
|
this.materializeConfigMountAfterBranchSwitch();
|
|
6486
7283
|
} finally {
|
|
6487
7284
|
release();
|
|
@@ -6490,7 +7287,8 @@ export class AgentFramework {
|
|
|
6490
7287
|
console.error(
|
|
6491
7288
|
`[operator] rollback agent=${agentName} to=${tailMessageId}` +
|
|
6492
7289
|
`${tailMessageId !== opts.messageId ? ` (requested ${opts.messageId}, snapped to end of body group)` : ''}` +
|
|
6493
|
-
` removed=${messagesRemoved} branch=${branchName}
|
|
7290
|
+
` removed=${messagesRemoved} branch=${branchName} marks=${describeMarkers(markers)}` +
|
|
7291
|
+
` by=${describeRequester(opts.requester)}`,
|
|
6494
7292
|
);
|
|
6495
7293
|
this.recordOperatorAction({
|
|
6496
7294
|
...logBase,
|
|
@@ -6500,6 +7298,7 @@ export class AgentFramework {
|
|
|
6500
7298
|
tailMessageId,
|
|
6501
7299
|
messagesRemoved,
|
|
6502
7300
|
discordRefs: removedRefs.length,
|
|
7301
|
+
markers,
|
|
6503
7302
|
},
|
|
6504
7303
|
});
|
|
6505
7304
|
return {
|
|
@@ -6509,6 +7308,7 @@ export class AgentFramework {
|
|
|
6509
7308
|
messagesRemoved,
|
|
6510
7309
|
tailMessageId,
|
|
6511
7310
|
removedRefs,
|
|
7311
|
+
markers,
|
|
6512
7312
|
lastVisible: await this.lastVisiblePreview(agentName),
|
|
6513
7313
|
};
|
|
6514
7314
|
} catch (error) {
|
|
@@ -6524,6 +7324,9 @@ export class AgentFramework {
|
|
|
6524
7324
|
* removed together. Discord originals get awareness markers via the outbox;
|
|
6525
7325
|
* the batch is `explicit`-activated only after every removal succeeded, and
|
|
6526
7326
|
* a crash mid-way is finished at next boot (resumePreparedDiscordSuppressions).
|
|
7327
|
+
* Like rollbackToMessage, it returns once the body change has landed, with
|
|
7328
|
+
* a marker-scheduling receipt (`markers`); delivery continues without the
|
|
7329
|
+
* caller waiting on it.
|
|
6527
7330
|
*
|
|
6528
7331
|
* Not retroactive over derived state: a message already folded into an
|
|
6529
7332
|
* autobiographical summary stays in that summary — roll back to before it
|
|
@@ -6545,6 +7348,8 @@ export class AgentFramework {
|
|
|
6545
7348
|
messagesRemoved: number;
|
|
6546
7349
|
removedIds: string[];
|
|
6547
7350
|
removedRefs: Array<{ serverId: string; channelId: string; messageId: string }>;
|
|
7351
|
+
/** Whether awareness marks were scheduled; never a delivery claim. */
|
|
7352
|
+
markers: SurgeryMarkerReceipt;
|
|
6548
7353
|
lastVisible: { participant?: string; role?: string; preview?: string } | null;
|
|
6549
7354
|
}> {
|
|
6550
7355
|
const requestedIds = [...new Set(opts.messageIds.map(String))];
|
|
@@ -6597,9 +7402,12 @@ export class AgentFramework {
|
|
|
6597
7402
|
throw new OperatorActionError('invalid', 'Suppression branch name must differ from the active branch');
|
|
6598
7403
|
}
|
|
6599
7404
|
// Gate + reserve the whole store (see reserveStoreForSurgery); held
|
|
6600
|
-
// until the fork is fully redacted
|
|
7405
|
+
// until the fork is fully redacted and its marker scheduling has been
|
|
7406
|
+
// recorded or reported, or the source is restored. Remote delivery is
|
|
7407
|
+
// not waited on.
|
|
6601
7408
|
const release = this.reserveStoreForSurgery('suppress', agentName);
|
|
6602
7409
|
let createdBranch: string;
|
|
7410
|
+
let markers: SurgeryMarkerReceipt;
|
|
6603
7411
|
try {
|
|
6604
7412
|
const markerBatch = this.discordAwarenessOutbox?.prepare({
|
|
6605
7413
|
agentName,
|
|
@@ -6640,7 +7448,6 @@ export class AgentFramework {
|
|
|
6640
7448
|
if (iv.group || iv.fromId !== iv.toId) cm.removeMessages(iv.fromId as MessageId, iv.toId as MessageId);
|
|
6641
7449
|
else cm.removeMessage(iv.fromId as MessageId);
|
|
6642
7450
|
}
|
|
6643
|
-
if (markerBatch) this.discordAwarenessOutbox!.activate(markerBatch.id);
|
|
6644
7451
|
} catch (error) {
|
|
6645
7452
|
// A partially suppressed branch is not safe to serve from. Keep it
|
|
6646
7453
|
// for diagnosis, put the agent back on the untouched source — and
|
|
@@ -6653,7 +7460,19 @@ export class AgentFramework {
|
|
|
6653
7460
|
{ cause: error },
|
|
6654
7461
|
);
|
|
6655
7462
|
}
|
|
6656
|
-
|
|
7463
|
+
// Every redaction committed: the body change has landed and the fork
|
|
7464
|
+
// is safe to serve from. Activating the marker batch is bookkeeping
|
|
7465
|
+
// about a world-side effect; if it fails, the suppression still
|
|
7466
|
+
// stands, and the receipt says what became of the marks: not
|
|
7467
|
+
// scheduled once the batch is retired, or unresolved if it can't be.
|
|
7468
|
+
// (The batch's resume journal has nothing left to do, since every
|
|
7469
|
+
// interval has committed, so retiring the batch gives up only the
|
|
7470
|
+
// marks.)
|
|
7471
|
+
markers = this.activateSurgeryMarkers(markerBatch, 'suppress', agentName);
|
|
7472
|
+
// Installs the MCPL data-plane gate synchronously, before the store is
|
|
7473
|
+
// released, and starts delivery without awaiting it (the first
|
|
7474
|
+
// reaction request may already be on the wire when this returns).
|
|
7475
|
+
this.startDiscordAwarenessDelivery('suppress', agentName);
|
|
6657
7476
|
this.materializeConfigMountAfterBranchSwitch();
|
|
6658
7477
|
} finally {
|
|
6659
7478
|
release();
|
|
@@ -6661,7 +7480,7 @@ export class AgentFramework {
|
|
|
6661
7480
|
|
|
6662
7481
|
console.error(
|
|
6663
7482
|
`[operator] suppress agent=${agentName} removed=${messagesRemoved} branch=${createdBranch} ` +
|
|
6664
|
-
`by=${describeRequester(opts.requester)}`,
|
|
7483
|
+
`marks=${describeMarkers(markers)} by=${describeRequester(opts.requester)}`,
|
|
6665
7484
|
);
|
|
6666
7485
|
const loggedIds = capIds(removedIds);
|
|
6667
7486
|
this.recordOperatorAction({
|
|
@@ -6673,6 +7492,7 @@ export class AgentFramework {
|
|
|
6673
7492
|
removedIds: loggedIds.ids,
|
|
6674
7493
|
...(loggedIds.truncated ? { removedIdsTruncated: true } : {}),
|
|
6675
7494
|
discordRefs: removedRefs.length,
|
|
7495
|
+
markers,
|
|
6676
7496
|
},
|
|
6677
7497
|
});
|
|
6678
7498
|
return {
|
|
@@ -6682,6 +7502,7 @@ export class AgentFramework {
|
|
|
6682
7502
|
messagesRemoved,
|
|
6683
7503
|
removedIds,
|
|
6684
7504
|
removedRefs,
|
|
7505
|
+
markers,
|
|
6685
7506
|
lastVisible: await this.lastVisiblePreview(agentName),
|
|
6686
7507
|
};
|
|
6687
7508
|
} catch (error) {
|
|
@@ -6690,6 +7511,108 @@ export class AgentFramework {
|
|
|
6690
7511
|
}
|
|
6691
7512
|
}
|
|
6692
7513
|
|
|
7514
|
+
/**
|
|
7515
|
+
* Record a surgery's prepared marker batch as active once its body change
|
|
7516
|
+
* has landed, and say what happened. Never throws: the body is already
|
|
7517
|
+
* applied, so a ledger failure here is reported in the receipt rather than
|
|
7518
|
+
* as a failed surgery an operator might retry. A batch that could not be
|
|
7519
|
+
* activated is retired, so no later reconciliation can deliver it behind a
|
|
7520
|
+
* receipt saying it was not scheduled (`not-scheduled`). If retiring fails
|
|
7521
|
+
* too, the receipt says only that the outcome is `unresolved`: the batch is
|
|
7522
|
+
* retained, and it may still be delivered. A rollback's batch
|
|
7523
|
+
* (activated by target branch) may be promoted by the very next
|
|
7524
|
+
* reconciliation, the one the delivery it starts runs; a suppression's
|
|
7525
|
+
* (activated explicitly) waits for the next start.
|
|
7526
|
+
*
|
|
7527
|
+
* A batch with no refs (a suppression journal with nothing addressable) is
|
|
7528
|
+
* still activated, which closes its resume journal. It schedules no marks,
|
|
7529
|
+
* so it reports `none` whatever the ledger does. If it can be neither
|
|
7530
|
+
* activated nor retired, its journal is left prepared and closes at the
|
|
7531
|
+
* next start: every interval it names has already committed, so the resume
|
|
7532
|
+
* skips them all and activates it.
|
|
7533
|
+
*/
|
|
7534
|
+
private activateSurgeryMarkers(
|
|
7535
|
+
batch: DiscordAwarenessBatch | null,
|
|
7536
|
+
verb: 'rollback' | 'suppress',
|
|
7537
|
+
agentName: string,
|
|
7538
|
+
): SurgeryMarkerReceipt {
|
|
7539
|
+
if (!batch) return { status: 'none', queued: 0 };
|
|
7540
|
+
try {
|
|
7541
|
+
this.discordAwarenessOutbox!.activate(batch.id);
|
|
7542
|
+
} catch (error) {
|
|
7543
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
7544
|
+
let retired = false;
|
|
7545
|
+
let retireDetail = '';
|
|
7546
|
+
try {
|
|
7547
|
+
retired = this.discordAwarenessOutbox!.discard(batch.id);
|
|
7548
|
+
} catch (discardError) {
|
|
7549
|
+
retireDetail = discardError instanceof Error ? discardError.message : String(discardError);
|
|
7550
|
+
}
|
|
7551
|
+
console.error(
|
|
7552
|
+
`[discord-awareness] ${verb} agent=${agentName}: body applied, but marker batch ${batch.id} ` +
|
|
7553
|
+
`could not be activated (${detail}); ` +
|
|
7554
|
+
(retired ? 'batch retired' : `batch retained${retireDetail ? ` (retire failed: ${retireDetail})` : ''}`),
|
|
7555
|
+
);
|
|
7556
|
+
if (batch.refs.length === 0) return { status: 'none', queued: 0 };
|
|
7557
|
+
this.opsAlert(
|
|
7558
|
+
retired ? 'discord-awareness-not-scheduled' : 'discord-awareness-unresolved',
|
|
7559
|
+
agentName,
|
|
7560
|
+
retired
|
|
7561
|
+
? `${verb} applied, but its ${batch.refs.length} awareness mark(s) were not scheduled ` +
|
|
7562
|
+
`(batch ${batch.id} retired): ${detail}`
|
|
7563
|
+
: `${verb} applied, but awareness batch ${batch.id} (${batch.refs.length} mark(s)) could be ` +
|
|
7564
|
+
`neither activated nor retired; it may still be delivered: ${detail}`,
|
|
7565
|
+
{ data: { batchId: batch.id, retired } },
|
|
7566
|
+
);
|
|
7567
|
+
return retired
|
|
7568
|
+
? { status: 'not-scheduled', queued: 0, error: detail }
|
|
7569
|
+
: { status: 'unresolved', queued: 0, batchId: batch.id, error: detail };
|
|
7570
|
+
}
|
|
7571
|
+
if (batch.refs.length === 0) return { status: 'none', queued: 0 };
|
|
7572
|
+
return { status: 'queued', queued: batch.refs.length, batchId: batch.id };
|
|
7573
|
+
}
|
|
7574
|
+
|
|
7575
|
+
/**
|
|
7576
|
+
* Start awareness delivery after a branch move (a surgery, a switch, an
|
|
7577
|
+
* undo or redo) without the caller awaiting it. No operator response waits
|
|
7578
|
+
* for Discord to accept marks: the MCPL data-plane gate alone holds channel
|
|
7579
|
+
* traffic that arrives after it until they are attempted.
|
|
7580
|
+
* `syncDiscordAwarenessMarkers` installs that gate synchronously, before
|
|
7581
|
+
* its first await, so calling this while a surgery still holds the store
|
|
7582
|
+
* puts the gate up before the store is released. With no drain running, the
|
|
7583
|
+
* drain starts in the same synchronous run (the first reaction request can
|
|
7584
|
+
* be written before this returns); with one running, this generation's pass
|
|
7585
|
+
* follows it (drainForGeneration). Either way delivery continues
|
|
7586
|
+
* independently: the caller's response and any store reservation stop
|
|
7587
|
+
* waiting on remote reactions. A turn may run during delivery when its wake
|
|
7588
|
+
* carries no channel traffic (a scheduler timer, a provider admission), and
|
|
7589
|
+
* also when its channel traffic was admitted before the gate went up
|
|
7590
|
+
* (queued when a surgery began, or arriving during its switch or fork): the
|
|
7591
|
+
* surgery's store reservation, which used to hold such a turn through the
|
|
7592
|
+
* drain, now ends once the gate is up. A failure is accounted by the gate
|
|
7593
|
+
* itself: the current generation's failure recycles the connections, and a
|
|
7594
|
+
* superseded one leaves that to the newer generation, whose own pass meets
|
|
7595
|
+
* the same ledger. Here it is logged and raised as an ops alert, and the
|
|
7596
|
+
* next sync, reconnect or start delivers what is still pending.
|
|
7597
|
+
*
|
|
7598
|
+
* @param after What moved the branch, for the log and the alert
|
|
7599
|
+
* ("rollback", "branch switch", "undo" …).
|
|
7600
|
+
* @param agentName The agent the move was for, or "framework" for a
|
|
7601
|
+
* branch switch that names none.
|
|
7602
|
+
*/
|
|
7603
|
+
startDiscordAwarenessDelivery(after: string, agentName = 'framework'): void {
|
|
7604
|
+
if (!this.discordAwarenessOutbox) return;
|
|
7605
|
+
this.syncDiscordAwarenessMarkers().catch((error) => {
|
|
7606
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
7607
|
+
console.error(`[discord-awareness] delivery after ${after} agent=${agentName} failed: ${detail}`);
|
|
7608
|
+
this.opsAlert(
|
|
7609
|
+
'discord-awareness-delivery',
|
|
7610
|
+
agentName,
|
|
7611
|
+
`Awareness mark delivery after ${after} failed: ${detail}`,
|
|
7612
|
+
);
|
|
7613
|
+
});
|
|
7614
|
+
}
|
|
7615
|
+
|
|
6693
7616
|
/** Re-materialize config files from the (new) active branch — fire and
|
|
6694
7617
|
* forget; the gate picks changes up via mtime. Mirrors undoLastTurn. */
|
|
6695
7618
|
private materializeConfigMountAfterBranchSwitch(): void {
|
|
@@ -7129,7 +8052,7 @@ export class AgentFramework {
|
|
|
7129
8052
|
const membraneResults = currentState.toolResults.map(tc =>
|
|
7130
8053
|
this.toMembraneToolResult(tc.id, tc.result, maxChars, spilled.get(tc.id))
|
|
7131
8054
|
);
|
|
7132
|
-
agent.toolResultGuard.storeResults(toolResultContent, membraneResults, currentState.toolResults, turnRowMetadata);
|
|
8055
|
+
agent.toolResultGuard.storeResults(toolResultContent, membraneResults, currentState.toolResults, turnRowMetadata, maxChars);
|
|
7133
8056
|
|
|
7134
8057
|
// Flush any messages that were deferred while this turn was in
|
|
7135
8058
|
// flight. Route to the PRIMARY agent — deferred messages are
|
|
@@ -7822,10 +8745,10 @@ export class AgentFramework {
|
|
|
7822
8745
|
isolate: true,
|
|
7823
8746
|
// Strategy instances are stateful — never share the template's.
|
|
7824
8747
|
strategy: router.strategyFactory?.() ?? new PassthroughStrategy(),
|
|
7825
|
-
//
|
|
7826
|
-
//
|
|
7827
|
-
//
|
|
7828
|
-
membrane: this.
|
|
8748
|
+
// Provider cooldown ownership belongs to the persistent resident only;
|
|
8749
|
+
// generation-unique forks must not leak gates. A fork's own recorded
|
|
8750
|
+
// provider waits still bind its compression calls.
|
|
8751
|
+
membrane: this.conversationAuxiliaryMembraneFor(name),
|
|
7829
8752
|
debugLogContext: !!process.env.DEBUG_CONTEXT,
|
|
7830
8753
|
});
|
|
7831
8754
|
|
|
@@ -7893,6 +8816,18 @@ export class AgentFramework {
|
|
|
7893
8816
|
this.conversationAgentHomes.delete(agentName);
|
|
7894
8817
|
this.evictTurnCheckpoints(agentName);
|
|
7895
8818
|
if (agent) this.logicalTurnToolCalls.delete(agent);
|
|
8819
|
+
// A provider-wait hold (nonOwningProviderWait) goes with the agent: left,
|
|
8820
|
+
// its timer would requeue held wakes for an agent that is gone, and a wait
|
|
8821
|
+
// held until released would re-arm for the name indefinitely. The recorded
|
|
8822
|
+
// wait stays, and binds the name if its channel binds again.
|
|
8823
|
+
const cooldown = this.providerAccelerationCooldowns.get(agentName);
|
|
8824
|
+
if (cooldown) {
|
|
8825
|
+
clearTimeout(cooldown.timer);
|
|
8826
|
+
this.providerAccelerationCooldowns.delete(agentName);
|
|
8827
|
+
console.error(`[provider-wait] agent=${agentName} disposed while held` +
|
|
8828
|
+
`${cooldown.waitModel !== undefined ? ` on ${cooldown.waitModel}` : ''}: ` +
|
|
8829
|
+
`${cooldown.heldRequests.length} held request(s) dropped with it (the recorded wait stays)`);
|
|
8830
|
+
}
|
|
7896
8831
|
this.emitTrace({
|
|
7897
8832
|
type: 'mcpl:conversation-disposed',
|
|
7898
8833
|
agentName,
|
|
@@ -8751,7 +9686,10 @@ export class AgentFramework {
|
|
|
8751
9686
|
|
|
8752
9687
|
const providerCooldown = this.providerAccelerationCooldowns?.get(agentName);
|
|
8753
9688
|
if (providerCooldown) {
|
|
8754
|
-
|
|
9689
|
+
// A provider wait binds its own model; with a different primary model
|
|
9690
|
+
// selected now, the turn proceeds (the wait stays recorded).
|
|
9691
|
+
const otherModel = providerCooldown.waitModel !== undefined && providerCooldown.waitModel !== agent.model;
|
|
9692
|
+
if (!otherModel && (now < providerCooldown.until || this.extendHostProviderHold(agentName, providerCooldown))) {
|
|
8755
9693
|
for (const req of requests) {
|
|
8756
9694
|
if (!providerCooldown.heldRequests.some((r) => this.sameInferenceRequest(r, req))) {
|
|
8757
9695
|
providerCooldown.heldRequests.push(req);
|
|
@@ -8770,6 +9708,12 @@ export class AgentFramework {
|
|
|
8770
9708
|
if (recovery) recovery.releasedAt = now;
|
|
8771
9709
|
requests = [...providerCooldown.heldRequests, ...requests];
|
|
8772
9710
|
}
|
|
9711
|
+
// A provider's stated wait on this agent's primary model, including one
|
|
9712
|
+
// recorded before a restart, holds the turn until it passes. An
|
|
9713
|
+
// ephemeral run records nothing, its name being single-use, so no wait,
|
|
9714
|
+
// known or unreadable, binds it (as at the last boundary).
|
|
9715
|
+
const providerWait = this.providerWaits?.active(agentName, agent.model);
|
|
9716
|
+
if (providerWait && !this.ephemeralRuns.has(agentName) && this.holdForProviderWait(agent, providerWait, requests)) continue;
|
|
8773
9717
|
|
|
8774
9718
|
// Skip if agent is busy (inferring, streaming, or waiting for tools) —
|
|
8775
9719
|
// or if a turn is alive at all (activeTurnTokens): the state machine
|
|
@@ -9537,6 +10481,12 @@ export class AgentFramework {
|
|
|
9537
10481
|
): Promise<void> {
|
|
9538
10482
|
const ownsProviderGate =
|
|
9539
10483
|
!this.ephemeralRuns.has(agent.name) && !this.conversationAgentHomes.has(agent.name);
|
|
10484
|
+
// A stopping framework starts no turn, whoever owns provider admission
|
|
10485
|
+
// (conversation agents and ephemeral runs do not take the gate).
|
|
10486
|
+
if (this.providerAdmissionClosed) {
|
|
10487
|
+
if (ownsProviderGate && providerGateAlreadyHeld) this.releasePrimaryProviderGate(agent.name);
|
|
10488
|
+
return;
|
|
10489
|
+
}
|
|
9540
10490
|
if (ownsProviderGate && !providerGateAlreadyHeld) {
|
|
9541
10491
|
this.acquirePrimaryProviderGate(agent.name);
|
|
9542
10492
|
const gate = this.providerGate(agent.name);
|
|
@@ -9583,7 +10533,7 @@ export class AgentFramework {
|
|
|
9583
10533
|
await this.startAgentStream(agent, trigger, attempt, true);
|
|
9584
10534
|
}).catch((error) => {
|
|
9585
10535
|
this.releasePrimaryProviderGate(agent.name);
|
|
9586
|
-
console.error(`[provider-cooldown] failed to resume primary for ${agent.name}
|
|
10536
|
+
console.error(`[provider-cooldown] failed to resume primary for ${agent.name}: ${describeFailureWithFrames(error)}`);
|
|
9587
10537
|
});
|
|
9588
10538
|
return;
|
|
9589
10539
|
}
|
|
@@ -9592,6 +10542,35 @@ export class AgentFramework {
|
|
|
9592
10542
|
return;
|
|
9593
10543
|
}
|
|
9594
10544
|
}
|
|
10545
|
+
// The last admission boundary before this primary call. With the gate
|
|
10546
|
+
// held and no auxiliary call in flight, no new provider wait can be
|
|
10547
|
+
// recorded for this agent before the call (auxiliary admission waits on
|
|
10548
|
+
// the gate). But one may have been recorded since the scheduler's check:
|
|
10549
|
+
// an auxiliary call this wake parked behind can fail with a stated wait
|
|
10550
|
+
// on the primary model. That wait binds this wake too: hold it, don't call.
|
|
10551
|
+
// A conversation agent's waits are recorded and held as a resident's are
|
|
10552
|
+
// (nonOwningProviderWait), and its retries and in-turn restarts reach this
|
|
10553
|
+
// boundary without the scheduler: a wait recorded meanwhile, by its own
|
|
10554
|
+
// auxiliary call or a non-retryable failure, binds them here too. An
|
|
10555
|
+
// ephemeral run records nothing, so nothing binds it here. A frame whose
|
|
10556
|
+
// agent was replaced or removed (a fork disposed while its retry slept)
|
|
10557
|
+
// installs no hold: its turn is dropped after the compile, as main drops
|
|
10558
|
+
// any such turn.
|
|
10559
|
+
if (!this.ephemeralRuns.has(agent.name) && this.agents.get(agent.name) === agent) {
|
|
10560
|
+
const wait = this.providerWaits?.active(agent.name, agent.model);
|
|
10561
|
+
if (wait && this.holdForProviderWait(agent, wait, trigger ? [trigger] : [])) {
|
|
10562
|
+
if (ownsProviderGate) this.releasePrimaryProviderGate(agent.name);
|
|
10563
|
+
// A retry or an in-turn restart arrives with its turn still open at
|
|
10564
|
+
// the event gate: the failed attempt, or the predecessor that kept
|
|
10565
|
+
// liveness for it, leaves closing it to this frame. Held here, no
|
|
10566
|
+
// frame would close it, and while it is open every gate event
|
|
10567
|
+
// buffers, for every agent, until the wait ends.
|
|
10568
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
10569
|
+
console.error(`[provider-wait] agent=${agent.name} model=${agent.model} binds at admission ` +
|
|
10570
|
+
`(recorded after the wake was scheduled); wake held, not started`);
|
|
10571
|
+
return;
|
|
10572
|
+
}
|
|
10573
|
+
}
|
|
9595
10574
|
// Mark the turn alive before ANYTHING awaits (hooks, compile, stream
|
|
9596
10575
|
// setup): from here until this turn's teardown, cross-turn writers defer
|
|
9597
10576
|
// (addMessage guard) instead of appending — nothing may enter the window
|
|
@@ -9967,6 +10946,28 @@ export class AgentFramework {
|
|
|
9967
10946
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
9968
10947
|
return false;
|
|
9969
10948
|
}
|
|
10949
|
+
// A conversation agent takes no provider gate, so its own compression
|
|
10950
|
+
// can record a wait while this turn compiles, after the last boundary
|
|
10951
|
+
// checked. Check its waits again here, where nothing has been sent, and
|
|
10952
|
+
// hold the turn as the boundary would rather than send it into the wait.
|
|
10953
|
+
if (this.conversationAgentHomes.has(agent.name)) {
|
|
10954
|
+
const wait = this.providerWaits?.active(agent.name, agent.model);
|
|
10955
|
+
if (wait && this.holdForProviderWait(agent, wait, trigger ? [trigger] : [])) {
|
|
10956
|
+
console.error(`[provider-wait] agent=${agent.name} model=${agent.model} binds after the compile ` +
|
|
10957
|
+
`(recorded while the turn compiled); turn held, nothing sent`);
|
|
10958
|
+
// The turn emitted inference:started before its compile; every turn
|
|
10959
|
+
// that does gets one terminal event, which listeners (a typing loop)
|
|
10960
|
+
// rely on, as the failure sites' holds emit inference:failed.
|
|
10961
|
+
this.emitTrace({ type: 'inference:failed', agentName: agent.name, error: providerWaitText(agent.name, agent.model, wait) });
|
|
10962
|
+
stream.cancel();
|
|
10963
|
+
agent.cancelStream();
|
|
10964
|
+
this.activeStreams.delete(agent.name);
|
|
10965
|
+
this.pendingAssistantBlocks.delete(agent.name);
|
|
10966
|
+
this.channelRegistry?.stopTyping();
|
|
10967
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
10968
|
+
return false;
|
|
10969
|
+
}
|
|
10970
|
+
}
|
|
9970
10971
|
|
|
9971
10972
|
const handle = this.driveStream(
|
|
9972
10973
|
agent,
|
|
@@ -10004,10 +11005,28 @@ export class AgentFramework {
|
|
|
10004
11005
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
10005
11006
|
return false;
|
|
10006
11007
|
}
|
|
11008
|
+
const statedWait = ownsProviderGate ? undefined : this.nonOwningProviderWait(agent, err, trigger);
|
|
11009
|
+
if (statedWait === 'held') {
|
|
11010
|
+
// A conversation agent held by the provider's stated wait, as a
|
|
11011
|
+
// resident is: the hold's release owns the later compile.
|
|
11012
|
+
this.eventGate?.onInferenceEnded(agent.name);
|
|
11013
|
+
return false;
|
|
11014
|
+
}
|
|
10007
11015
|
|
|
10008
|
-
const action = this.errorPolicy.onInferenceError(err, agent.name, attempt);
|
|
11016
|
+
const action: ErrorAction = statedWait === 'fail' ? { retry: false } : this.errorPolicy.onInferenceError(err, agent.name, attempt);
|
|
10009
11017
|
if (action.retry) {
|
|
10010
|
-
|
|
11018
|
+
// A policy's delay can be a provider's retry-after: wait all of it.
|
|
11019
|
+
await this.waitForRetry(action.delayMs);
|
|
11020
|
+
if (this.providerAdmissionClosed) {
|
|
11021
|
+
// stop() ended the wait: that cancels the retry, it does not hasten
|
|
11022
|
+
// it. Nothing is written (the store is closing); this frame still
|
|
11023
|
+
// owns its token, since no retry frame replaced it.
|
|
11024
|
+
if (this.activeTurnTokens.get(agent.name) === turnToken) {
|
|
11025
|
+
this.activeTurnTokens.delete(agent.name);
|
|
11026
|
+
this.activeTurnTriggers.delete(agent.name);
|
|
11027
|
+
}
|
|
11028
|
+
return false;
|
|
11029
|
+
}
|
|
10011
11030
|
// The retry re-enters startAgentStream, which replaces this frame's
|
|
10012
11031
|
// turn token with its own; cleanup belongs to the innermost frame.
|
|
10013
11032
|
await this.startAgentStream(agent, trigger, attempt + 1);
|
|
@@ -10024,16 +11043,18 @@ export class AgentFramework {
|
|
|
10024
11043
|
this.settleAgent(agent.name, {
|
|
10025
11044
|
stopReason: 'exhausted',
|
|
10026
11045
|
speech: '',
|
|
10027
|
-
error: err.message,
|
|
11046
|
+
error: statedWait === 'fail' ? this.statedWaitNotSlept(err) : err.message,
|
|
10028
11047
|
}, startLogical);
|
|
10029
11048
|
this.emitTrace({
|
|
10030
11049
|
type: 'inference:exhausted',
|
|
10031
11050
|
agentName: agent.name,
|
|
10032
11051
|
error: err.message,
|
|
10033
|
-
// Drives the poison-history breaker (
|
|
10034
|
-
//
|
|
10035
|
-
//
|
|
10036
|
-
//
|
|
11052
|
+
// Drives the poison-history breaker (shedMayHelp: an
|
|
11053
|
+
// `invalid_request` at HTTP 400, 413 or 422 or with no status, or a
|
|
11054
|
+
// `context_length` at 413, suggests the provider refused the
|
|
11055
|
+
// request's content or size, which a shed of the newest exchange may
|
|
11056
|
+
// fix; auth/abort and other context_length failures are also
|
|
11057
|
+
// non-retryable but must NOT cost history) and the OverBudget drain breaker
|
|
10037
11058
|
// (`over_budget`: compile refused to fit the hard budget — this is
|
|
10038
11059
|
// the site that sees it, since compile runs before the stream
|
|
10039
11060
|
// exists). `retryable` is kept for observability.
|
|
@@ -10669,7 +11690,7 @@ export class AgentFramework {
|
|
|
10669
11690
|
);
|
|
10670
11691
|
agent.toolResultGuard.storeResults(toolResultContent, readyState.toolResults.map((tc) =>
|
|
10671
11692
|
this.toMembraneToolResult(tc.id, tc.result, cap, spilled.get(tc.id))), readyState.toolResults,
|
|
10672
|
-
this.silentTurnRowMetadata(agent.name));
|
|
11693
|
+
this.silentTurnRowMetadata(agent.name), cap);
|
|
10673
11694
|
// The response is already complete: this batch is never
|
|
10674
11695
|
// submitted in this turn, so it cannot be refused. Admit it.
|
|
10675
11696
|
agent.toolResultGuard.settleTurnEnded();
|
|
@@ -10931,7 +11952,7 @@ export class AgentFramework {
|
|
|
10931
11952
|
// border is close, rather than noise on every near-threshold
|
|
10932
11953
|
// flip. (It must not re-enter any agent's context — the emoji
|
|
10933
11954
|
// is suppressed at the surface via
|
|
10934
|
-
//
|
|
11955
|
+
// DISCORD_SUPPRESSED_REACTIONS_BASELINE; without that, marking a
|
|
10935
11956
|
// refusal feeds the next one.)
|
|
10936
11957
|
if (!handledByRewind) {
|
|
10937
11958
|
void this.reactToRefusal(agent.name, category);
|
|
@@ -11212,25 +12233,37 @@ export class AgentFramework {
|
|
|
11212
12233
|
if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) this.eventGate?.onInferenceEnded(agent.name);
|
|
11213
12234
|
break;
|
|
11214
12235
|
}
|
|
12236
|
+
const statedWait = ownsProviderGate ? undefined : this.nonOwningProviderWait(agent, err, trigger);
|
|
12237
|
+
if (statedWait === 'held') {
|
|
12238
|
+
lifecyclePhase = 'failed';
|
|
12239
|
+
if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) this.eventGate?.onInferenceEnded(agent.name);
|
|
12240
|
+
break;
|
|
12241
|
+
}
|
|
11215
12242
|
|
|
11216
|
-
const action = this.errorPolicy.onInferenceError(err, agent.name, attempt);
|
|
12243
|
+
const action: ErrorAction = statedWait === 'fail' ? { retry: false } : this.errorPolicy.onInferenceError(err, agent.name, attempt);
|
|
11217
12244
|
if (action.retry) {
|
|
11218
|
-
|
|
11219
|
-
await this.
|
|
12245
|
+
// A policy's delay can be a provider's retry-after: wait all of it.
|
|
12246
|
+
await this.waitForRetry(action.delayMs);
|
|
12247
|
+
// stop() ending the wait cancels the retry (this frame's finally
|
|
12248
|
+
// clears the token); it is never permission to retry at once.
|
|
12249
|
+
if (!this.providerAdmissionClosed) await this.startAgentStream(agent, trigger, attempt + 1);
|
|
11220
12250
|
} else {
|
|
11221
12251
|
this.settleAgent(agent.name, {
|
|
11222
12252
|
stopReason: 'exhausted',
|
|
11223
12253
|
speech: '',
|
|
11224
|
-
error: err.message,
|
|
12254
|
+
error: statedWait === 'fail' ? this.statedWaitNotSlept(err) : err.message,
|
|
11225
12255
|
}, this.agents.get(agent.name) === agent && agent.streamId === myStreamId ? streamRouting.logical : undefined);
|
|
11226
12256
|
this.emitTrace({
|
|
11227
12257
|
type: 'inference:exhausted',
|
|
11228
12258
|
agentName: agent.name,
|
|
11229
12259
|
error: err.message,
|
|
11230
12260
|
// Drives the poison-history breaker. `retryable` is kept for
|
|
11231
|
-
// observability, but the
|
|
11232
|
-
//
|
|
11233
|
-
//
|
|
12261
|
+
// observability, but the breaker gates on
|
|
12262
|
+
// shedMayHelp(errorType, httpStatus): only an `invalid_request`
|
|
12263
|
+
// at HTTP 400, 413 or 422 or with no status, or a
|
|
12264
|
+
// `context_length` at 413, suggests the provider refused the
|
|
12265
|
+
// request's content or size, which a shed of the newest
|
|
12266
|
+
// exchange may fix.
|
|
11234
12267
|
...this.classifyInferenceError(err),
|
|
11235
12268
|
});
|
|
11236
12269
|
if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) this.eventGate?.onInferenceEnded(agent.name);
|
|
@@ -11456,6 +12489,20 @@ export class AgentFramework {
|
|
|
11456
12489
|
lifecyclePhase = 'failed'; agent.reset(); if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) this.eventGate?.onInferenceEnded(agent.name);
|
|
11457
12490
|
return;
|
|
11458
12491
|
}
|
|
12492
|
+
// The same rule for a turn that doesn't own provider admission as at
|
|
12493
|
+
// the other two failure sites: a conversation agent's stated wait is
|
|
12494
|
+
// recorded and its turn held; an ephemeral run whose wait it won't sleep
|
|
12495
|
+
// says why it ends.
|
|
12496
|
+
const statedWait = ownsProviderGate ? undefined : this.nonOwningProviderWait(agent, err, trigger);
|
|
12497
|
+
if (statedWait === 'held') {
|
|
12498
|
+
this.emitTrace({ type: 'inference:failed', agentName: agent.name, error: err.message, stack: err.stack });
|
|
12499
|
+
this.logInference({ timestamp: startTime, agentName: agent.name, requestId, success: false,
|
|
12500
|
+
error: `Provider wait: ${err.message}`,
|
|
12501
|
+
request: compiledRequest ?? { note: 'streaming request rate-limited' }, durationMs });
|
|
12502
|
+
this.abortAgentScript(agent.name, 'provider wait');
|
|
12503
|
+
lifecyclePhase = 'failed'; agent.reset(); if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) this.eventGate?.onInferenceEnded(agent.name);
|
|
12504
|
+
return;
|
|
12505
|
+
}
|
|
11459
12506
|
this.emitTrace({
|
|
11460
12507
|
type: 'inference:failed',
|
|
11461
12508
|
agentName: agent.name,
|
|
@@ -11465,7 +12512,7 @@ export class AgentFramework {
|
|
|
11465
12512
|
this.settleAgent(agent.name, {
|
|
11466
12513
|
stopReason: 'exhausted',
|
|
11467
12514
|
speech: '',
|
|
11468
|
-
error: err.message,
|
|
12515
|
+
error: statedWait === 'fail' ? this.statedWaitNotSlept(err) : err.message,
|
|
11469
12516
|
}, streamRouting.logical);
|
|
11470
12517
|
this.emitTrace({
|
|
11471
12518
|
type: 'inference:exhausted',
|
|
@@ -12897,6 +13944,9 @@ export class AgentFramework {
|
|
|
12897
13944
|
private logInference(entry: InferenceLogEntry): void {
|
|
12898
13945
|
// Store large request/response as blobs
|
|
12899
13946
|
const entryToStore = { ...entry };
|
|
13947
|
+
// The failure text is stored inline in every record; the request that
|
|
13948
|
+
// produced it is already stored separately as a blob.
|
|
13949
|
+
if (typeof entryToStore.error === 'string') entryToStore.error = boundFailureText(entryToStore.error);
|
|
12900
13950
|
|
|
12901
13951
|
// Blob threshold: 10KB - typical context-heavy requests exceed this
|
|
12902
13952
|
const BLOB_THRESHOLD = 10000;
|
|
@@ -13481,6 +14531,10 @@ export class AgentFramework {
|
|
|
13481
14531
|
(event.error as string) ?? 'unknown error',
|
|
13482
14532
|
event.retryable as boolean | undefined,
|
|
13483
14533
|
event.errorType as string | undefined,
|
|
14534
|
+
{
|
|
14535
|
+
httpStatus: event.httpStatus as number | undefined,
|
|
14536
|
+
providerErrorCode: event.providerErrorCode as string | undefined,
|
|
14537
|
+
},
|
|
13484
14538
|
);
|
|
13485
14539
|
} else if (event.type === 'inference:completed') {
|
|
13486
14540
|
// A successful response — even mid-turn between tool calls — proves the
|
|
@@ -13499,8 +14553,26 @@ export class AgentFramework {
|
|
|
13499
14553
|
...event,
|
|
13500
14554
|
timestamp: Date.now(),
|
|
13501
14555
|
} as TraceEvent;
|
|
13502
|
-
|
|
13503
|
-
|
|
14556
|
+
// Listeners (web UI, subagent observers, host loggers) receive bounded
|
|
14557
|
+
// failure text. An event's `error` and `stack` strings are whatever its
|
|
14558
|
+
// source's Error carried, which for a provider failure can be an echo of
|
|
14559
|
+
// the whole request.
|
|
14560
|
+
const bounded = traceEvent as unknown as Record<string, unknown>;
|
|
14561
|
+
if (typeof bounded.error === 'string') bounded.error = boundFailureText(bounded.error);
|
|
14562
|
+
if (typeof bounded.stack === 'string') bounded.stack = boundFailureText(bounded.stack);
|
|
14563
|
+
|
|
14564
|
+
// An event goes to the subscriptions present when its delivery starts,
|
|
14565
|
+
// each one only if it is still active when its turn comes, as the DOM's
|
|
14566
|
+
// EventTarget delivers: a subscription made during delivery waits for the
|
|
14567
|
+
// next event, and one removed during delivery is skipped if its turn
|
|
14568
|
+
// hasn't come. Each onTrace call is a subscription of its own, so a
|
|
14569
|
+
// function subscribed twice is called twice. Iterating the live array let
|
|
14570
|
+
// an unsubscribe during delivery make the listener after the one being
|
|
14571
|
+
// called miss the event, since splice shifts the rest under the iterator.
|
|
14572
|
+
for (const registration of [...this.traceListeners]) {
|
|
14573
|
+
if (!registration.active) continue;
|
|
14574
|
+
// Called as a plain function, as before, so the registration isn't its `this`.
|
|
14575
|
+
const { listener } = registration;
|
|
13504
14576
|
try {
|
|
13505
14577
|
listener(traceEvent);
|
|
13506
14578
|
} catch (error) {
|
|
@@ -13523,15 +14595,18 @@ export class AgentFramework {
|
|
|
13523
14595
|
* inference, so this never causes a retry/wake loop.
|
|
13524
14596
|
* 3. escalation — after N consecutive failures the agent is hard-down;
|
|
13525
14597
|
* log that loudly (a repeated identical failure is the textbook signal).
|
|
13526
|
-
* 4. breaker — when
|
|
13527
|
-
*
|
|
13528
|
-
*
|
|
13529
|
-
*
|
|
13530
|
-
*
|
|
13531
|
-
*
|
|
13532
|
-
*
|
|
13533
|
-
*
|
|
13534
|
-
* the
|
|
14598
|
+
* 4. breaker — when a failure at or past the hard-down threshold suggests
|
|
14599
|
+
* the provider refused the request's content or size (shedMayHelp:
|
|
14600
|
+
* an `invalid_request` at HTTP 400, 413 or 422 or with no status, or a
|
|
14601
|
+
* `context_length` at 413, such as a provider's answer to a corrupted
|
|
14602
|
+
* tool_use/tool_result pairing or an oversized attachment), retrying
|
|
14603
|
+
* the same context is likely to fail the same way, and every new push
|
|
14604
|
+
* event would wake the agent onto that same history. At the hard-down
|
|
14605
|
+
* threshold, automatically quarantine: shed the newest complete
|
|
14606
|
+
* exchange (the same forced-rewind primitive `/unstick` uses —
|
|
14607
|
+
* shedNewestTurn never orphans a tool_use/thinking block) and retry,
|
|
14608
|
+
* bounded by the same rewind cap so the breaker can never eat the
|
|
14609
|
+
* whole history.
|
|
13535
14610
|
*/
|
|
13536
14611
|
/**
|
|
13537
14612
|
* Live health snapshot for /healthz and doctor tooling: gate state, queued
|
|
@@ -13556,11 +14631,14 @@ export class AgentFramework {
|
|
|
13556
14631
|
primaryPending: gate?.primaryPending ?? false,
|
|
13557
14632
|
auxiliaryInFlight: gate?.auxiliaryInFlight ?? 0,
|
|
13558
14633
|
auxiliaryDeferred: gate?.deferredAuxiliary ?? 0,
|
|
13559
|
-
|
|
14634
|
+
// Null while none, and while a provider wait holds until released.
|
|
14635
|
+
cooldownUntil: cooldown !== undefined && Number.isFinite(cooldown.until) ? cooldown.until : null,
|
|
13560
14636
|
cooldownReason: cooldown?.reason ?? null,
|
|
13561
14637
|
hostHold: cooldown?.hostHoldError !== undefined,
|
|
13562
14638
|
heldRequests: cooldown?.heldRequests.length ?? 0,
|
|
13563
14639
|
lastRecovery: this.providerAccelerationLastRecovery?.get(name) ?? null,
|
|
14640
|
+
/** Provider waits (stated retry-after) binding this agent now, per model. */
|
|
14641
|
+
providerWaits: this.providerWaitSnapshot(name).map(({ agent: _agent, ...wait }) => wait),
|
|
13564
14642
|
};
|
|
13565
14643
|
})(),
|
|
13566
14644
|
lastInference: this.lastInferenceAt.get(name) ?? null,
|
|
@@ -13576,10 +14654,14 @@ export class AgentFramework {
|
|
|
13576
14654
|
* and kind-specific fields (additive only — doctor parses by regex). */
|
|
13577
14655
|
private logFailure(record: Record<string, unknown>): void {
|
|
13578
14656
|
try {
|
|
14657
|
+
// Every field is bounded: `reason` is conventionally free text, and any
|
|
14658
|
+
// caller's field may carry an error message, or an Error whose own
|
|
14659
|
+
// fields include the whole request.
|
|
14660
|
+
const bounded = boundFailureFields(record);
|
|
13579
14661
|
mkdirSync('logs', { recursive: true });
|
|
13580
14662
|
appendFileSync(
|
|
13581
14663
|
'logs/failures.log',
|
|
13582
|
-
JSON.stringify({ at: new Date().toISOString(), ...
|
|
14664
|
+
JSON.stringify({ at: new Date().toISOString(), ...bounded }) + '\n',
|
|
13583
14665
|
);
|
|
13584
14666
|
} catch { /* best-effort */ }
|
|
13585
14667
|
}
|
|
@@ -13612,13 +14694,27 @@ export class AgentFramework {
|
|
|
13612
14694
|
private opsAlert(
|
|
13613
14695
|
kind: string,
|
|
13614
14696
|
agentName: string,
|
|
13615
|
-
|
|
14697
|
+
rawMessage: string,
|
|
13616
14698
|
opts?: { data?: Record<string, unknown>; skipLog?: boolean },
|
|
13617
14699
|
): void {
|
|
14700
|
+
// A JavaScript caller of notifyOps or notifyOpsAlert can pass a message
|
|
14701
|
+
// that isn't a string; it's described as a thrown value is, rather than
|
|
14702
|
+
// thrown on (a string is its own description).
|
|
14703
|
+
const wholeMessage = failureDescription(rawMessage);
|
|
14704
|
+
const message = boundFailureText(wholeMessage);
|
|
14705
|
+
// The caller's data goes to failures.log and to trace listeners, bounded
|
|
14706
|
+
// the same way for both. A JavaScript caller's null is no data, and data
|
|
14707
|
+
// that isn't a plain object (a string, an array, a Buffer) is kept as one
|
|
14708
|
+
// field, `data`, rather than spread into a field per character or element.
|
|
14709
|
+
const rawData: unknown = opts?.data;
|
|
14710
|
+
const plain = typeof rawData === 'object' && rawData !== null &&
|
|
14711
|
+
[Object.prototype, null].includes(Object.getPrototypeOf(rawData));
|
|
14712
|
+
const data = rawData == null ? undefined
|
|
14713
|
+
: boundFailureFields(plain ? rawData as Record<string, unknown> : { data: rawData });
|
|
13618
14714
|
if (!opts?.skipLog) {
|
|
13619
|
-
this.logFailure({ agent: agentName, kind, reason: message, ...
|
|
14715
|
+
this.logFailure({ agent: agentName, kind, reason: message, ...data });
|
|
13620
14716
|
}
|
|
13621
|
-
this.emitTrace({ type: 'ops:alert', kind, agentName, message, data
|
|
14717
|
+
this.emitTrace({ type: 'ops:alert', kind, agentName, message, data });
|
|
13622
14718
|
|
|
13623
14719
|
const hook = process.env.CONNECTOME_OPS_WEBHOOK;
|
|
13624
14720
|
if (!hook) return;
|
|
@@ -13631,7 +14727,7 @@ export class AgentFramework {
|
|
|
13631
14727
|
method: 'POST',
|
|
13632
14728
|
headers: { 'content-type': 'application/json' },
|
|
13633
14729
|
body: JSON.stringify({
|
|
13634
|
-
content: `\u{1F6A8} **${agentName}** ${kind}: ${
|
|
14730
|
+
content: `\u{1F6A8} **${agentName}** ${kind}: ${boundFailureText(wholeMessage, OPS_WEBHOOK_MESSAGE_MAX_CHARS)}`,
|
|
13635
14731
|
}),
|
|
13636
14732
|
}).catch((err) => console.error(
|
|
13637
14733
|
`[ops-alert] webhook post failed (${key}):`,
|
|
@@ -13684,9 +14780,12 @@ export class AgentFramework {
|
|
|
13684
14780
|
|
|
13685
14781
|
/**
|
|
13686
14782
|
* Classify a terminal inference error for the `inference:exhausted` trace.
|
|
13687
|
-
* Single-sourced for every emit site
|
|
13688
|
-
*
|
|
13689
|
-
*
|
|
14783
|
+
* Single-sourced for every emit site that has an error to classify (a
|
|
14784
|
+
* stream the framework aborted has none): the downstream breakers gate on
|
|
14785
|
+
* what it returns (the poison-history quarantine on `errorType` and
|
|
14786
|
+
* `httpStatus`, through shedMayHelp; the OverBudget drain kick on
|
|
14787
|
+
* `errorType`), so classification drift between sites would silently
|
|
14788
|
+
* change when a safety net acts.
|
|
13690
14789
|
*
|
|
13691
14790
|
* context-manager's refusal errors are matched by a real `instanceof` since
|
|
13692
14791
|
* CM exports them from its package root (context-manager#41 / #71; floor
|
|
@@ -13697,9 +14796,23 @@ export class AgentFramework {
|
|
|
13697
14796
|
* Deliberately NOT a message match: the message wording belongs to CM and
|
|
13698
14797
|
* can be reworded without warning.
|
|
13699
14798
|
*/
|
|
13700
|
-
private classifyInferenceError(err: Error): {
|
|
13701
|
-
|
|
13702
|
-
|
|
14799
|
+
private classifyInferenceError(err: Error): {
|
|
14800
|
+
retryable?: boolean; errorType?: string; httpStatus?: number; providerErrorCode?: string;
|
|
14801
|
+
} {
|
|
14802
|
+
// From either copy of membrane (isMembraneError). The fields are read as
|
|
14803
|
+
// data, and the type is bounded like describeFailure's, for every sink the
|
|
14804
|
+
// classification reaches (the trace, the marker, the logs).
|
|
14805
|
+
if (isMembraneError(err)) {
|
|
14806
|
+
const fields = err as Error & {
|
|
14807
|
+
retryable?: unknown; type?: unknown; httpStatus?: unknown; providerErrorCode?: unknown;
|
|
14808
|
+
};
|
|
14809
|
+
const providerErrorCode = boundProviderErrorCode(fields.providerErrorCode);
|
|
14810
|
+
return {
|
|
14811
|
+
...(typeof fields.retryable === 'boolean' ? { retryable: fields.retryable } : {}),
|
|
14812
|
+
...(typeof fields.type === 'string' ? { errorType: boundFailureText(fields.type, PROVIDER_ERROR_CODE_MAX_CHARS) } : {}),
|
|
14813
|
+
...(typeof fields.httpStatus === 'number' ? { httpStatus: fields.httpStatus } : {}),
|
|
14814
|
+
...(providerErrorCode !== undefined ? { providerErrorCode } : {}),
|
|
14815
|
+
};
|
|
13703
14816
|
}
|
|
13704
14817
|
if (err instanceof OverBudgetError || err.name === 'OverBudgetError') {
|
|
13705
14818
|
return { errorType: 'over_budget' };
|
|
@@ -13727,25 +14840,70 @@ export class AgentFramework {
|
|
|
13727
14840
|
|
|
13728
14841
|
private noteInferenceExhausted(
|
|
13729
14842
|
agentName: string,
|
|
13730
|
-
|
|
14843
|
+
rawReason: string,
|
|
13731
14844
|
retryable?: boolean,
|
|
13732
|
-
|
|
14845
|
+
rawErrorType?: string,
|
|
14846
|
+
detail?: { httpStatus?: number; providerErrorCode?: string },
|
|
13733
14847
|
): void {
|
|
14848
|
+
// The reason is the failure's own message, and a provider can echo the
|
|
14849
|
+
// whole rejected request in it: one household's [inference-failed]
|
|
14850
|
+
// marker carried ~1.7 MB of its own request back into its context. Every
|
|
14851
|
+
// sink below gets an excerpt cut from the whole reason at its own bound
|
|
14852
|
+
// (boundFailureText, so its marker states the whole length): the stderr
|
|
14853
|
+
// line, failures.log and the hard-down alert FAILURE_TEXT_MAX_CHARS, the
|
|
14854
|
+
// [inference-failed] marker FAILURE_MARKER_REASON_MAX_CHARS, the status's
|
|
14855
|
+
// lastError and the compile-refused alert FAILURE_SUMMARY_MAX_CHARS, and
|
|
14856
|
+
// the rewind marker REWIND_MARKER_REASON_MAX_CHARS. The classification
|
|
14857
|
+
// travels as structured fields beside the text.
|
|
14858
|
+
//
|
|
14859
|
+
// A reason that isn't a string (the stream paths always pass a message)
|
|
14860
|
+
// is described rather than allowed to throw before anything is recorded.
|
|
14861
|
+
// Its whole description is the whole reason, cut once by each sink.
|
|
14862
|
+
const wholeReason: string = typeof rawReason === 'string' ? rawReason : failureDescription(rawReason);
|
|
14863
|
+
// The type is bounded wherever it's written, as classifyInferenceError
|
|
14864
|
+
// bounds the types it reads, since a caller can pass one it didn't classify.
|
|
14865
|
+
const errorType = typeof rawErrorType === 'string'
|
|
14866
|
+
? boundFailureText(rawErrorType, PROVIDER_ERROR_CODE_MAX_CHARS) : rawErrorType;
|
|
14867
|
+
const reason = boundFailureText(wholeReason);
|
|
14868
|
+
const markerReason = boundFailureText(wholeReason, FAILURE_MARKER_REASON_MAX_CHARS);
|
|
14869
|
+
const httpStatus = typeof detail?.httpStatus === 'number' ? detail.httpStatus : undefined;
|
|
14870
|
+
const providerErrorCode = boundProviderErrorCode(detail?.providerErrorCode);
|
|
14871
|
+
const classification: Record<string, unknown> = {
|
|
14872
|
+
...(errorType !== undefined ? { errorType } : {}),
|
|
14873
|
+
...(retryable !== undefined ? { retryable } : {}),
|
|
14874
|
+
...(httpStatus !== undefined ? { httpStatus } : {}),
|
|
14875
|
+
...(providerErrorCode !== undefined ? { providerErrorCode } : {}),
|
|
14876
|
+
};
|
|
14877
|
+
// Each excerpt says how long the whole was when it was cut.
|
|
14878
|
+
const cutAt = (excerpt: string): Record<string, unknown> =>
|
|
14879
|
+
excerpt === wholeReason ? {} : { reasonChars: wholeReason.length };
|
|
13734
14880
|
const streak = (this.consecutiveInferenceFailures.get(agentName) ?? 0) + 1;
|
|
13735
14881
|
this.consecutiveInferenceFailures.set(agentName, streak);
|
|
13736
|
-
this.lastInferenceAt.set(agentName, { ...this.lastInferenceAt.get(agentName), failedAt: Date.now(), lastError:
|
|
14882
|
+
this.lastInferenceAt.set(agentName, { ...this.lastInferenceAt.get(agentName), failedAt: Date.now(), lastError: boundFailureText(wholeReason, FAILURE_SUMMARY_MAX_CHARS) });
|
|
13737
14883
|
|
|
13738
14884
|
// (1) Durable stderr line — works in headless/daemon mode with no client.
|
|
13739
|
-
|
|
14885
|
+
const labels = [
|
|
14886
|
+
errorType !== undefined ? `type=${errorType}` : undefined,
|
|
14887
|
+
httpStatus !== undefined ? `status=${httpStatus}` : undefined,
|
|
14888
|
+
providerErrorCode !== undefined ? `code=${providerErrorCode}` : undefined,
|
|
14889
|
+
].filter((label): label is string => label !== undefined);
|
|
14890
|
+
console.error(
|
|
14891
|
+
`[inference-failed] agent=${agentName} consecutive=${streak}` +
|
|
14892
|
+
`${labels.length > 0 ? ` ${labels.join(' ')}` : ''}: ${reason}`,
|
|
14893
|
+
);
|
|
13740
14894
|
|
|
13741
14895
|
// (1b) Machine-greppable durable record, independent of journald/unit log
|
|
13742
14896
|
// redirects: logs/failures.log under the host's working directory. This is
|
|
13743
|
-
// what connectome-doctor reads. Legacy fields kept; `kind`
|
|
13744
|
-
|
|
14897
|
+
// what connectome-doctor reads. Legacy fields kept; `kind` and the
|
|
14898
|
+
// classification fields are additive.
|
|
14899
|
+
this.logFailure({ agent: agentName, consecutive: streak, reason, kind: 'inference-exhausted', ...classification, ...cutAt(reason) });
|
|
13745
14900
|
|
|
13746
14901
|
// (2) Agent-facing chronicle marker (no inference triggered → no loop).
|
|
13747
14902
|
const agent = this.agents.get(agentName);
|
|
13748
14903
|
if (agent && process.env.SUPPRESS_INFERENCE_FAILED_MARKER !== '1') {
|
|
14904
|
+
const cause = [errorType, httpStatus !== undefined ? `HTTP ${httpStatus}` : undefined]
|
|
14905
|
+
.filter((part): part is string => part !== undefined)
|
|
14906
|
+
.join(', ');
|
|
13749
14907
|
try {
|
|
13750
14908
|
agent.getContextManager().addMessage(
|
|
13751
14909
|
'user',
|
|
@@ -13753,15 +14911,15 @@ export class AgentFramework {
|
|
|
13753
14911
|
type: 'text',
|
|
13754
14912
|
text:
|
|
13755
14913
|
`[inference-failed] Your previous turn did not complete: the model ` +
|
|
13756
|
-
`call failed and produced no response, so nothing was sent. Reason: ` +
|
|
13757
|
-
`${
|
|
14914
|
+
`call failed${cause ? ` (${cause})` : ''} and produced no response, so nothing was sent. Reason: ` +
|
|
14915
|
+
`${markerReason}. If this recurs with the same cause, change approach ` +
|
|
13758
14916
|
`rather than retrying identically (e.g. drop an oversized attachment ` +
|
|
13759
14917
|
`or an unsupported setting).`,
|
|
13760
14918
|
}],
|
|
13761
|
-
{ system: true, kind: 'inference-failed', reason, consecutive: streak },
|
|
14919
|
+
{ system: true, kind: 'inference-failed', reason: markerReason, consecutive: streak, ...classification, ...cutAt(markerReason) },
|
|
13762
14920
|
);
|
|
13763
14921
|
} catch (err) {
|
|
13764
|
-
console.error(`[inference-failed] could not record chronicle marker for ${agentName}
|
|
14922
|
+
console.error(`[inference-failed] could not record chronicle marker for ${agentName}: ${describeFailureWithFrames(err)}`);
|
|
13765
14923
|
}
|
|
13766
14924
|
}
|
|
13767
14925
|
|
|
@@ -13780,10 +14938,16 @@ export class AgentFramework {
|
|
|
13780
14938
|
// (e.g. a reason string that crossed a serialization boundary). It matches
|
|
13781
14939
|
// CM's current OverBudgetError wording and MAY rot if CM rewords it — the
|
|
13782
14940
|
// errorType gate is the one that's load-bearing.
|
|
14941
|
+
//
|
|
14942
|
+
// The fallback applies only to an unclassified reason. A classified
|
|
14943
|
+
// failure (a provider's 400, say) already says what it is, and its
|
|
14944
|
+
// message is the provider's text, which can quote anything, including
|
|
14945
|
+
// these phrases from the agent's own context.
|
|
13783
14946
|
const overBudget =
|
|
13784
14947
|
errorType === 'over_budget' ||
|
|
13785
14948
|
errorType === 'context_refusal' ||
|
|
13786
|
-
|
|
14949
|
+
// Decided on the whole reason: presentation bounds never change it.
|
|
14950
|
+
(errorType === undefined && /exceed hard budget|no summary covers/i.test(wholeReason));
|
|
13787
14951
|
// Observability: a refused compile means the agent cannot think AT ALL
|
|
13788
14952
|
// this turn — surface it on the ops-alert pipeline (ops:alert trace +
|
|
13789
14953
|
// webhook → fleet-watch) immediately, not only at the hard-down streak.
|
|
@@ -13793,7 +14957,7 @@ export class AgentFramework {
|
|
|
13793
14957
|
this.opsAlert(
|
|
13794
14958
|
'context-refusal',
|
|
13795
14959
|
agentName,
|
|
13796
|
-
`compile refused (${errorType ?? 'unclassified'}): ${
|
|
14960
|
+
`compile refused (${errorType ?? 'unclassified'}): ${boundFailureText(wholeReason, FAILURE_SUMMARY_MAX_CHARS)} — ` +
|
|
13797
14961
|
`drain breaker ${agent ? 'kicking' : 'unavailable (no agent handle)'}; ` +
|
|
13798
14962
|
`if this persists, raise contextBudgetTokens or overBudgetGraceRatio.`,
|
|
13799
14963
|
{ skipLog: true },
|
|
@@ -13811,7 +14975,7 @@ export class AgentFramework {
|
|
|
13811
14975
|
}
|
|
13812
14976
|
console.error(`[inference-failed] drain kicked for ${agentName} (OverBudget breaker, ${ticks} ticks)`);
|
|
13813
14977
|
} catch (err) {
|
|
13814
|
-
console.error(`[inference-failed] drain kick failed for ${agentName} after ${ticks} ticks
|
|
14978
|
+
console.error(`[inference-failed] drain kick failed for ${agentName} after ${ticks} ticks: ${describeFailureWithFrames(err)}`);
|
|
13815
14979
|
} finally {
|
|
13816
14980
|
this.overBudgetDrainInFlight.delete(agentName);
|
|
13817
14981
|
}
|
|
@@ -13828,24 +14992,32 @@ export class AgentFramework {
|
|
|
13828
14992
|
// Ops alert (webhook + ops:alert trace), throttled per (agent, kind) —
|
|
13829
14993
|
// a hard-down agent re-posts every ~15 min, not on every failed retry.
|
|
13830
14994
|
// failures.log already got the per-exhaustion record above.
|
|
14995
|
+
// The excerpt leaves room for the prefix, so the alert's own bound
|
|
14996
|
+
// never cuts it a second time.
|
|
14997
|
+
const hardDown = `${streak} consecutive inference failures. Last reason: `;
|
|
13831
14998
|
this.opsAlert(
|
|
13832
14999
|
'hard-down',
|
|
13833
15000
|
agentName,
|
|
13834
|
-
|
|
15001
|
+
hardDown + boundFailureText(wholeReason, FAILURE_TEXT_MAX_CHARS - hardDown.length),
|
|
13835
15002
|
{ skipLog: true },
|
|
13836
15003
|
);
|
|
13837
15004
|
|
|
13838
|
-
// (4) Poison-history breaker: ONLY
|
|
13839
|
-
//
|
|
13840
|
-
//
|
|
13841
|
-
//
|
|
13842
|
-
//
|
|
13843
|
-
//
|
|
13844
|
-
//
|
|
13845
|
-
//
|
|
15005
|
+
// (4) Poison-history breaker: ONLY when shedMayHelp(errorType,
|
|
15006
|
+
// httpStatus) holds, for an `invalid_request` at HTTP 400, 413 or 422 or
|
|
15007
|
+
// with no status, or a `context_length` at 413, which suggests the
|
|
15008
|
+
// provider refused the request's content or size. The current failure
|
|
15009
|
+
// decides, whatever failed before it in the streak. Deliberately NOT
|
|
15010
|
+
// keyed on `retryable === false`, which membrane also returns for auth
|
|
15011
|
+
// (expired key), abort (deliberate cancel), a context_length at 400 or
|
|
15012
|
+
// 422 (a token overflow, which the context budget exists to prevent;
|
|
15013
|
+
// shedding the newest exchange is the wrong direction for it), safety
|
|
15014
|
+
// and unsupported. For none of them is shedding the newest exchange
|
|
15015
|
+
// likely to help, and each would otherwise shed good exchanges and
|
|
15016
|
+
// stamp a rewind marker saying the API kept refusing them.
|
|
15017
|
+
// `retryable` is kept only for the trace.
|
|
13846
15018
|
void retryable;
|
|
13847
|
-
if (
|
|
13848
|
-
this.quarantinePoisonedHistory(agent,
|
|
15019
|
+
if (agent && shedMayHelp(errorType, httpStatus)) {
|
|
15020
|
+
this.quarantinePoisonedHistory(agent, wholeReason);
|
|
13849
15021
|
}
|
|
13850
15022
|
}
|
|
13851
15023
|
}
|
|
@@ -13856,14 +15028,14 @@ export class AgentFramework {
|
|
|
13856
15028
|
* primitives as the refusal auto-rewind / `/unstick` (shedNewestTurn +
|
|
13857
15029
|
* the single consolidated episode marker) — and queues a retry so the
|
|
13858
15030
|
* rewound history is verified immediately instead of waiting for the next
|
|
13859
|
-
* push event to wake the agent onto the same
|
|
15031
|
+
* push event to wake the agent onto the same context.
|
|
13860
15032
|
*
|
|
13861
15033
|
* Bounded: at most `refusalHandling.maxRewinds` (default 3, hard cap 10)
|
|
13862
15034
|
* sheds per failure episode; the budget resets on any successful inference.
|
|
13863
15035
|
* At the cap (or with nothing left to shed) it stops — the agent stays up
|
|
13864
15036
|
* and hard-down logging continues, but no further history is consumed.
|
|
13865
15037
|
*/
|
|
13866
|
-
private quarantinePoisonedHistory(agent: Agent,
|
|
15038
|
+
private quarantinePoisonedHistory(agent: Agent, wholeReason: string): void {
|
|
13867
15039
|
const cap = Math.max(1, Math.min(10, agent.refusalHandling?.maxRewinds ?? 3));
|
|
13868
15040
|
const used = this.exhaustionRewinds.get(agent.name) ?? 0;
|
|
13869
15041
|
if (used >= cap) {
|
|
@@ -13884,7 +15056,8 @@ export class AgentFramework {
|
|
|
13884
15056
|
}
|
|
13885
15057
|
|
|
13886
15058
|
this.exhaustionRewinds.set(agent.name, used + 1);
|
|
13887
|
-
const count = this.updateRewindMarker(
|
|
15059
|
+
const count = this.updateRewindMarker(
|
|
15060
|
+
agent, boundFailureText(wholeReason, REWIND_MARKER_REASON_MAX_CHARS), 'inference-failure');
|
|
13888
15061
|
console.error(
|
|
13889
15062
|
`[inference-rewind] agent=${agent.name} auto-quarantined ${rec.descriptor} ` +
|
|
13890
15063
|
`(${count} turn(s) shed this episode, cap ${cap}) after non-retryable ` +
|
|
@@ -14124,7 +15297,11 @@ export class AgentFramework {
|
|
|
14124
15297
|
await this.refreshMcplTools();
|
|
14125
15298
|
}
|
|
14126
15299
|
|
|
14127
|
-
/**
|
|
15300
|
+
/**
|
|
15301
|
+
* Reconcile the durable ledger with Chronicle, then deliver every server's
|
|
15302
|
+
* work. A drain still running is waited for and then followed by a fresh
|
|
15303
|
+
* pass (drainForGeneration).
|
|
15304
|
+
*/
|
|
14128
15305
|
async syncDiscordAwarenessMarkers(_onlyServerId?: string): Promise<void> {
|
|
14129
15306
|
if (!this.discordAwarenessOutbox) return;
|
|
14130
15307
|
// A targeted retry may discover pending work for other Discord servers
|
|
@@ -14339,46 +15516,70 @@ export class AgentFramework {
|
|
|
14339
15516
|
throw new DiscordAwarenessAccountingError('branch reconciliation', error);
|
|
14340
15517
|
}
|
|
14341
15518
|
const pending = this.readDiscordAwarenessPending();
|
|
14342
|
-
|
|
15519
|
+
// A drain that is still running holds operations it read before this
|
|
15520
|
+
// reconciliation, and it sends them whatever the ledger now says. A branch
|
|
15521
|
+
// move can hide them from `pending`: restoring a rollback's source makes
|
|
15522
|
+
// its marks undesired, and an add whose reply has not arrived is not yet
|
|
15523
|
+
// present, so the ledger shows nothing to do while the drain goes on to
|
|
15524
|
+
// send its saved adds and then the removals they need. So a generation
|
|
15525
|
+
// accounts for every running drain as well as for every pending
|
|
15526
|
+
// operation, and waits for those drains instead of releasing the data
|
|
15527
|
+
// planes beside them (drainForGeneration).
|
|
15528
|
+
const serverIds = [...new Set([
|
|
15529
|
+
...pending.map((operation) => operation.ref.serverId),
|
|
15530
|
+
...this.discordAwarenessDrains.keys(),
|
|
15531
|
+
])];
|
|
15532
|
+
if (serverIds.length === 0) {
|
|
14343
15533
|
return {
|
|
14344
15534
|
requiresBarrier: false,
|
|
14345
15535
|
promise: Promise.resolve({ status: 'delivered', delivered: 0, failed: 0 }),
|
|
14346
15536
|
};
|
|
14347
15537
|
}
|
|
14348
|
-
const serverIds = [...new Set(pending.map((operation) => operation.ref.serverId))];
|
|
14349
15538
|
const promise = Promise.all(
|
|
14350
|
-
serverIds.map((serverId) => this.
|
|
14351
|
-
).then(
|
|
14352
|
-
if (outcomes.every((outcome) => outcome.status === 'unavailable')) {
|
|
14353
|
-
return {
|
|
14354
|
-
status: 'unavailable',
|
|
14355
|
-
accounted: outcomes.reduce(
|
|
14356
|
-
(count, outcome) => count + (outcome.status === 'unavailable' ? outcome.accounted : 0),
|
|
14357
|
-
0,
|
|
14358
|
-
),
|
|
14359
|
-
};
|
|
14360
|
-
}
|
|
14361
|
-
return {
|
|
14362
|
-
status: 'delivered',
|
|
14363
|
-
delivered: outcomes.reduce(
|
|
14364
|
-
(count, outcome) => count + (outcome.status === 'delivered' ? outcome.delivered : 0),
|
|
14365
|
-
0,
|
|
14366
|
-
),
|
|
14367
|
-
failed: outcomes.reduce(
|
|
14368
|
-
(count, outcome) => count + (outcome.status === 'delivered' ? outcome.failed : 0),
|
|
14369
|
-
0,
|
|
14370
|
-
),
|
|
14371
|
-
};
|
|
14372
|
-
});
|
|
15539
|
+
serverIds.map((serverId) => this.drainForGeneration(serverId)),
|
|
15540
|
+
).then(combineDrainOutcomes);
|
|
14373
15541
|
return { requiresBarrier: true, promise };
|
|
14374
15542
|
}
|
|
14375
15543
|
|
|
15544
|
+
/**
|
|
15545
|
+
* One server's drain for a new generation: a gate completes only after
|
|
15546
|
+
* every operation pending when it was armed has been attempted. A drain
|
|
15547
|
+
* already running can't promise that. It never re-sends a key it has
|
|
15548
|
+
* attempted, so an operation re-pended while it runs (a rollback, its undo,
|
|
15549
|
+
* then a redo, all mid-drain) stays pending, and it may already have made
|
|
15550
|
+
* its last read of the ledger, missing a batch activated since. So a
|
|
15551
|
+
* generation that finds a drain running waits for it, then drains again
|
|
15552
|
+
* with a fresh pass that reads the ledger anew. Two generations waiting on
|
|
15553
|
+
* the same drain share the fresh pass that follows it.
|
|
15554
|
+
*/
|
|
15555
|
+
private drainForGeneration(serverId: string): Promise<DiscordAwarenessDrainOutcome> {
|
|
15556
|
+
const running = this.discordAwarenessDrains.get(serverId);
|
|
15557
|
+
if (!running) return this.drainDiscordAwarenessOutbox(serverId);
|
|
15558
|
+
return running.then((first) => this.drainDiscordAwarenessOutbox(serverId)
|
|
15559
|
+
.then((fresh) => combineDrainOutcomes([first, fresh])));
|
|
15560
|
+
}
|
|
15561
|
+
|
|
15562
|
+
/**
|
|
15563
|
+
* Wait out the awareness gate for an inbound event: the generation current
|
|
15564
|
+
* now, and any generation installed while waiting, until none is current
|
|
15565
|
+
* or the current one is one already waited for. A handler that waited only
|
|
15566
|
+
* for the generation it first saw would go on behind a newer one armed
|
|
15567
|
+
* meanwhile. Rejects as the generation it waited for rejects.
|
|
15568
|
+
*/
|
|
15569
|
+
private async awaitDiscordAwarenessBarriers(): Promise<void> {
|
|
15570
|
+
let waited: DiscordAwarenessBarrier | null = null;
|
|
15571
|
+
for (let barrier = this.discordAwarenessBarrier; barrier && barrier !== waited; barrier = this.discordAwarenessBarrier) {
|
|
15572
|
+
waited = barrier;
|
|
15573
|
+
await barrier.promise;
|
|
15574
|
+
}
|
|
15575
|
+
}
|
|
15576
|
+
|
|
14376
15577
|
/**
|
|
14377
15578
|
* Install one framework-global Discord-awareness generation before releasing
|
|
14378
|
-
* any inference-bearing event. With pending work
|
|
14379
|
-
* paused while all control planes remain live.
|
|
14380
|
-
* runs synchronously so request responders preserve
|
|
14381
|
-
* same-stack behavior.
|
|
15579
|
+
* any inference-bearing event. With pending work or a drain still running,
|
|
15580
|
+
* every MCPL data plane is paused while all control planes remain live.
|
|
15581
|
+
* With neither, ready() runs synchronously so request responders preserve
|
|
15582
|
+
* their historical same-stack behavior.
|
|
14382
15583
|
*/
|
|
14383
15584
|
private installMcplDataPlaneGate(): DiscordAwarenessBarrier {
|
|
14384
15585
|
for (const connection of this.mcplServerRegistry?.getAllServers() ?? []) {
|
|
@@ -14481,6 +15682,14 @@ export class AgentFramework {
|
|
|
14481
15682
|
throw new Error('MCPL subsystem is not initialized');
|
|
14482
15683
|
}
|
|
14483
15684
|
|
|
15685
|
+
// Every connect needs a readable awareness ledger: a runtime connect's
|
|
15686
|
+
// awareness barrier reconciles it, and a stdio child taking the
|
|
15687
|
+
// protective default derives that from it. Read it first, so an
|
|
15688
|
+
// unreadable ledger refuses the connect before anything is registered
|
|
15689
|
+
// under its id or spawned (a reconnecting server would otherwise be left
|
|
15690
|
+
// registered as a closed stub). With no outbox configured nothing is read.
|
|
15691
|
+
this.getPlacedReactionBaseline();
|
|
15692
|
+
|
|
14484
15693
|
// Register prefix + config for tool dispatch routing (idempotent with
|
|
14485
15694
|
// the pre-registration pass in initializeMcpl).
|
|
14486
15695
|
const prefix = config.toolPrefix ?? `mcpl--${config.id}`;
|
|
@@ -14496,7 +15705,33 @@ export class AgentFramework {
|
|
|
14496
15705
|
);
|
|
14497
15706
|
}
|
|
14498
15707
|
|
|
14499
|
-
|
|
15708
|
+
// A stdio child's protective reaction default, DISCORD_SUPPRESSED_REACTIONS_BASELINE
|
|
15709
|
+
// (WebSocket servers own their environment outside this process). The
|
|
15710
|
+
// caller's config stays unchanged, so a runtime restart derives afresh.
|
|
15711
|
+
// The default is derived at each spawn: the env below is a getter, which
|
|
15712
|
+
// the transport reads at first connect, at every automatic reconnect and
|
|
15713
|
+
// at a stub's retry, so retained markers added since the last spawn are
|
|
15714
|
+
// covered.
|
|
15715
|
+
let connectionConfig = config;
|
|
15716
|
+
if (!isWebSocketTransport(config)) {
|
|
15717
|
+
const declared = this.declaredReactionBaseline(config);
|
|
15718
|
+
if (declared) {
|
|
15719
|
+
console.error(
|
|
15720
|
+
`[mcpl] ${config.id}: DISCORD_SUPPRESSED_REACTIONS_BASELINE ${declared.source === 'env' ? 'from its env' : 'inherited from the host'} ` +
|
|
15721
|
+
`replaces the framework's protective default`,
|
|
15722
|
+
);
|
|
15723
|
+
}
|
|
15724
|
+
connectionConfig = { ...config };
|
|
15725
|
+
Object.defineProperty(connectionConfig, 'env', {
|
|
15726
|
+
enumerable: true,
|
|
15727
|
+
configurable: true,
|
|
15728
|
+
get: () => ({
|
|
15729
|
+
...config.env,
|
|
15730
|
+
DISCORD_SUPPRESSED_REACTIONS_BASELINE: declared?.value ?? this.getPlacedReactionBaseline().join(','),
|
|
15731
|
+
}),
|
|
15732
|
+
});
|
|
15733
|
+
}
|
|
15734
|
+
const connection = await this.mcplServerRegistry.addServer(connectionConfig, this.mcplHostCapabilities);
|
|
14500
15735
|
|
|
14501
15736
|
// Wire listeners before either startup staging or the runtime global gate
|
|
14502
15737
|
// releases control traffic needed for registration and marker service.
|
|
@@ -14862,6 +16097,92 @@ export class AgentFramework {
|
|
|
14862
16097
|
// Runtime MCPL server lifecycle (agent-facing hot deploy/restart/unload)
|
|
14863
16098
|
// ==========================================================================
|
|
14864
16099
|
|
|
16100
|
+
/**
|
|
16101
|
+
* Snapshot the protective Discord reaction default for this deployment, as
|
|
16102
|
+
* suppression tokens: the forms the Discord adapter's suppression matcher
|
|
16103
|
+
* reads, not marker payloads for an add, a remove or a replay. It covers
|
|
16104
|
+
* the reactions the framework places: refusal markers, the adapter's own
|
|
16105
|
+
* default awareness marker, the configured awareness marker, and retained
|
|
16106
|
+
* outbox markers (including offline recovery choices and marks that can
|
|
16107
|
+
* later be removed or replayed). A custom emoji appears as its bare ID,
|
|
16108
|
+
* which suppression matches even after a rename; stored markers and the
|
|
16109
|
+
* actual add/remove payloads stay unchanged. Explicit adapter or operator
|
|
16110
|
+
* suppression settings still take precedence. Throws on an unreadable
|
|
16111
|
+
* ledger rather than reporting a partial baseline.
|
|
16112
|
+
*/
|
|
16113
|
+
getPlacedReactionBaseline(): readonly string[] {
|
|
16114
|
+
let retained: string[];
|
|
16115
|
+
try {
|
|
16116
|
+
retained = this.discordAwarenessOutbox?.batches().map((batch) => batch.emoji) ?? [];
|
|
16117
|
+
} catch (error) {
|
|
16118
|
+
throw new DiscordAwarenessAccountingError('reaction-baseline ledger read', error);
|
|
16119
|
+
}
|
|
16120
|
+
// Discord events carry the custom emoji's name and ID separately. Full
|
|
16121
|
+
// marker forms belong in add/remove calls, but suppression matches their
|
|
16122
|
+
// stable ID (including after a rename), rather than the composite token.
|
|
16123
|
+
const suppressionToken = (emoji: string): string => {
|
|
16124
|
+
const custom = /^(?:<a?:\w+:(\d{17,20})>|(?:a:)?\w+:(\d{17,20}))$/.exec(emoji.trim());
|
|
16125
|
+
return custom?.[1] ?? custom?.[2] ?? emoji;
|
|
16126
|
+
};
|
|
16127
|
+
return [...new Set([
|
|
16128
|
+
...REFUSAL_REACTION_BASELINE,
|
|
16129
|
+
DEFAULT_DISCORD_AWARENESS_EMOJI,
|
|
16130
|
+
this.discordAwarenessEmoji,
|
|
16131
|
+
...retained,
|
|
16132
|
+
].map(suppressionToken))].filter((emoji) => emoji.length > 0);
|
|
16133
|
+
}
|
|
16134
|
+
|
|
16135
|
+
/**
|
|
16136
|
+
* The reaction baseline an operator chose for a stdio server, if any: a
|
|
16137
|
+
* value declared in its env, or one inherited from the host with
|
|
16138
|
+
* `inheritEnv`, the declared one first. A value with no tokens, as
|
|
16139
|
+
* discord-mcpl's parser reads them, counts as unset: the adapter splits on
|
|
16140
|
+
* commas and keeps a token only when its normalizer leaves something of it
|
|
16141
|
+
* (blank, `:`, `::` and a lone U+FE0F leave nothing), so such a value (an
|
|
16142
|
+
* empty systemd `Environment=`, say, or `${A},${B}` or `:${NAME}:` with the
|
|
16143
|
+
* variables empty) would otherwise turn the protective default off without
|
|
16144
|
+
* anyone choosing that.
|
|
16145
|
+
*
|
|
16146
|
+
* Windows names are matched in both sources: Workers expose a case-
|
|
16147
|
+
* sensitive `process.env` copy. Within one source, Node's spawn keeps the
|
|
16148
|
+
* first spelling in sorted order, so each source is resolved that way.
|
|
16149
|
+
*
|
|
16150
|
+
* A value is read as the child will see it. A config loaded from JSON can
|
|
16151
|
+
* carry a non-string there despite the type, and spawn writes every
|
|
16152
|
+
* defined value as its string form, so a number is a declared value, not a
|
|
16153
|
+
* failed connect. A null counts as unset, like a blank: written in JSON it
|
|
16154
|
+
* means no value, and read as spawn's "null" it would replace the
|
|
16155
|
+
* protective default with a list that suppresses nothing.
|
|
16156
|
+
*/
|
|
16157
|
+
private declaredReactionBaseline(
|
|
16158
|
+
config: import('./mcpl/types.js').McplServerConfig,
|
|
16159
|
+
): { value: string; source: 'env' | 'inherited' } | undefined {
|
|
16160
|
+
// A token as discord-mcpl's parser keeps one: `parseSuppressionEnvTokens`
|
|
16161
|
+
// trims each comma-separated token and keeps it when
|
|
16162
|
+
// `normalizeReactionEmoji` (discord-mcpl `src/filters.ts`) leaves
|
|
16163
|
+
// something, after it strips U+FE0F, trims, and strips one leading and one
|
|
16164
|
+
// trailing colon. The same steps in the same order, so the two can be
|
|
16165
|
+
// compared when either changes.
|
|
16166
|
+
const isToken = (token: string): boolean =>
|
|
16167
|
+
token.trim().replace(/\uFE0F/g, '').trim().replace(/^:|:$/g, '') !== '';
|
|
16168
|
+
const valueIn = (env: NodeJS.ProcessEnv | undefined): string | undefined => {
|
|
16169
|
+
let raw: unknown;
|
|
16170
|
+
if (process.platform !== 'win32') raw = env?.DISCORD_SUPPRESSED_REACTIONS_BASELINE;
|
|
16171
|
+
else {
|
|
16172
|
+
const key = Object.keys(env ?? {}).sort()
|
|
16173
|
+
.find((name) => name.toUpperCase() === 'DISCORD_SUPPRESSED_REACTIONS_BASELINE');
|
|
16174
|
+
raw = key === undefined ? undefined : env?.[key];
|
|
16175
|
+
}
|
|
16176
|
+
if (raw === undefined || raw === null) return undefined;
|
|
16177
|
+
const value = String(raw);
|
|
16178
|
+
return value.split(',').some(isToken) ? value : undefined;
|
|
16179
|
+
};
|
|
16180
|
+
const declared = valueIn(config.env);
|
|
16181
|
+
if (declared !== undefined) return { value: declared, source: 'env' };
|
|
16182
|
+
const inherited = config.inheritEnv ? valueIn(process.env) : undefined;
|
|
16183
|
+
return inherited !== undefined ? { value: inherited, source: 'inherited' } : undefined;
|
|
16184
|
+
}
|
|
16185
|
+
|
|
14865
16186
|
/**
|
|
14866
16187
|
* Connect a new MCPL server at runtime. Refreshes the tool list and
|
|
14867
16188
|
* notifies the agent of newly available tools. Throws if the MCPL
|
|
@@ -14944,6 +16265,12 @@ export class AgentFramework {
|
|
|
14944
16265
|
if (!config) {
|
|
14945
16266
|
throw new Error(`MCPL server "${id}" is not configured`);
|
|
14946
16267
|
}
|
|
16268
|
+
// Every connect reads the awareness ledger: its awareness barrier
|
|
16269
|
+
// reconciles it, and a stdio child taking the protective default derives
|
|
16270
|
+
// that from it. Read it before disconnecting, so an unreadable ledger
|
|
16271
|
+
// refuses the restart and leaves the running child up, whatever the
|
|
16272
|
+
// server's baseline or transport. (No outbox configured: nothing to read.)
|
|
16273
|
+
this.getPlacedReactionBaseline();
|
|
14947
16274
|
await this.disconnectMcplServer(id);
|
|
14948
16275
|
await this.connectMcplServer(config);
|
|
14949
16276
|
}
|
|
@@ -15073,8 +16400,10 @@ export class AgentFramework {
|
|
|
15073
16400
|
) => {
|
|
15074
16401
|
const answer = answerOnce(responder, 'push/event', connection.id);
|
|
15075
16402
|
try {
|
|
15076
|
-
|
|
15077
|
-
|
|
16403
|
+
// Only when a generation is current: with none, the event is handled in
|
|
16404
|
+
// the turn it arrives, as it always was (the push coalescer's ordering of
|
|
16405
|
+
// a push against its retraction depends on that).
|
|
16406
|
+
if (this.discordAwarenessBarrier) await this.awaitDiscordAwarenessBarriers();
|
|
15078
16407
|
// Checked here, after the barrier and before anything records the
|
|
15079
16408
|
// push as accepted: a stopped host refuses it.
|
|
15080
16409
|
if (this.queue?.isClosed) {
|
|
@@ -15104,8 +16433,7 @@ export class AgentFramework {
|
|
|
15104
16433
|
if (this.inferenceRouter && responder) {
|
|
15105
16434
|
const answer = answerOnce(responder, 'inference/request', connection.id);
|
|
15106
16435
|
try {
|
|
15107
|
-
|
|
15108
|
-
if (barrier) await barrier.promise;
|
|
16436
|
+
if (this.discordAwarenessBarrier) await this.awaitDiscordAwarenessBarriers();
|
|
15109
16437
|
// A stopping host starts no inference for a server: refused, like a
|
|
15110
16438
|
// push, before the router runs.
|
|
15111
16439
|
if (this.queue?.isClosed) {
|
|
@@ -15220,8 +16548,7 @@ export class AgentFramework {
|
|
|
15220
16548
|
) => {
|
|
15221
16549
|
const answer = answerOnce(responder, 'channels/incoming', connection.id);
|
|
15222
16550
|
try {
|
|
15223
|
-
|
|
15224
|
-
if (barrier) await barrier.promise;
|
|
16551
|
+
if (this.discordAwarenessBarrier) await this.awaitDiscordAwarenessBarriers();
|
|
15225
16552
|
if (this.queue?.isClosed) {
|
|
15226
16553
|
this.refuseServerItem('channels/incoming', connection.id, answer, { kind: 'refused' });
|
|
15227
16554
|
return;
|
|
@@ -15240,8 +16567,7 @@ export class AgentFramework {
|
|
|
15240
16567
|
if (!responder) return;
|
|
15241
16568
|
const answer = answerOnce(responder, 'host/command', connection.id);
|
|
15242
16569
|
try {
|
|
15243
|
-
|
|
15244
|
-
if (barrier) await barrier.promise;
|
|
16570
|
+
if (this.discordAwarenessBarrier) await this.awaitDiscordAwarenessBarriers();
|
|
15245
16571
|
// A stopping host runs no host command: refused like the requests
|
|
15246
16572
|
// above, before its handler runs.
|
|
15247
16573
|
if (this.queue?.isClosed) {
|
|
@@ -15288,7 +16614,7 @@ export class AgentFramework {
|
|
|
15288
16614
|
// grant) before the data-plane barrier releases — otherwise the first
|
|
15289
16615
|
// post-reconnect events land while the grant is still empty and are
|
|
15290
16616
|
// rejected fail-closed instead of delivered.
|
|
15291
|
-
|
|
16617
|
+
const reregistered = (async () => {
|
|
15292
16618
|
try {
|
|
15293
16619
|
const config = this.mcplServerConfigs.get(connection.id);
|
|
15294
16620
|
if (config) {
|
|
@@ -15303,7 +16629,11 @@ export class AgentFramework {
|
|
|
15303
16629
|
this.releaseMcplDataPlaneGate(awarenessBarrier);
|
|
15304
16630
|
this.handleToolsListChanged(connection.id);
|
|
15305
16631
|
})();
|
|
15306
|
-
|
|
16632
|
+
// The gate completes, and the data planes go ready, only once both the
|
|
16633
|
+
// awareness drain and the re-registration have finished. A drain can
|
|
16634
|
+
// settle first (one already running, or little to send), and the grant
|
|
16635
|
+
// must exist before the first data-plane event flows.
|
|
16636
|
+
void Promise.all([awarenessBarrier.promise, reregistered]).then(() => {
|
|
15307
16637
|
if (
|
|
15308
16638
|
awarenessBarrier.requiresBarrier
|
|
15309
16639
|
&& !this.completeMcplDataPlaneGate(awarenessBarrier)
|
|
@@ -16423,6 +17753,7 @@ export class AgentFramework {
|
|
|
16423
17753
|
| {
|
|
16424
17754
|
kind: 'reference';
|
|
16425
17755
|
refId: string;
|
|
17756
|
+
/** The type as the stub shows it (a label), lowercased: `image/…`. */
|
|
16426
17757
|
mediaType: string;
|
|
16427
17758
|
toolCallId: string;
|
|
16428
17759
|
toolName: string;
|
|
@@ -16476,10 +17807,22 @@ export class AgentFramework {
|
|
|
16476
17807
|
};
|
|
16477
17808
|
}
|
|
16478
17809
|
if (candidate.kind === 'reference') {
|
|
17810
|
+
// Point at fetch_reference only where it can deliver: not for a
|
|
17811
|
+
// reference this process holds no record of (fetch_reference
|
|
17812
|
+
// looks the id up in the same registry), nor, as the stub does,
|
|
17813
|
+
// for an unfetched one the fetcher would refuse as expired.
|
|
17814
|
+
const record = referenceRegistry.get(candidate.refId);
|
|
17815
|
+
const expiry = record && !record.fetchedPath ? referenceExpiry(record.testimony) : null;
|
|
16479
17816
|
throw new Error(
|
|
16480
17817
|
`save_recent_image: image at index ${position} is a reference (${candidate.refId}, ${candidate.mediaType}, ` +
|
|
16481
17818
|
`from ${describeTool(candidate)}) — its bytes are not held here. Nothing written. ` +
|
|
16482
|
-
|
|
17819
|
+
(!record
|
|
17820
|
+
? 'This process holds no record of it (a restart or an eviction drops records, and a stub quoted ' +
|
|
17821
|
+
"from elsewhere never had one), so fetch_reference can't fetch it either. If its tool can produce " +
|
|
17822
|
+
'it again, re-run that tool for a fresh reference.'
|
|
17823
|
+
: expiry
|
|
17824
|
+
? `It can't be fetched either — ${EXPIRY_REASON[expiry]}.`
|
|
17825
|
+
: 'Fetch it with fetch_reference and save the materialized file instead.'),
|
|
16483
17826
|
);
|
|
16484
17827
|
}
|
|
16485
17828
|
if (candidate.ref === null) {
|
|
@@ -16591,13 +17934,15 @@ export class AgentFramework {
|
|
|
16591
17934
|
consider({ kind: 'tool', ref: ph.ref, toolCallId, toolName, mediaType: ph.mediaType, messagesBack });
|
|
16592
17935
|
continue;
|
|
16593
17936
|
}
|
|
16594
|
-
// Reference stub: the registry's
|
|
16595
|
-
// still has the record)
|
|
16596
|
-
//
|
|
16597
|
-
//
|
|
16598
|
-
|
|
16599
|
-
|
|
16600
|
-
|
|
17937
|
+
// Reference stub: the registry's claimed type (when this process
|
|
17938
|
+
// still has the record), else the first image/ token in the
|
|
17939
|
+
// stub's text, which can come from its name, says whether it was
|
|
17940
|
+
// an image. Either is read as the stub shows a type, sanitized
|
|
17941
|
+
// and bounded. Non-image references are not slots.
|
|
17942
|
+
const claimed = referenceRegistry.get(ph.refId)?.testimony.mimeType ?? ph.mediaType ?? undefined;
|
|
17943
|
+
const mediaType = referenceTypeLabel(claimed).toLowerCase();
|
|
17944
|
+
if (!mediaType.startsWith('image/')) continue;
|
|
17945
|
+
consider({ kind: 'reference', refId: ph.refId, mediaType, toolCallId, toolName, messagesBack });
|
|
16601
17946
|
}
|
|
16602
17947
|
};
|
|
16603
17948
|
|
|
@@ -16650,10 +17995,14 @@ export class AgentFramework {
|
|
|
16650
17995
|
};
|
|
16651
17996
|
if (block?.type === 'image') {
|
|
16652
17997
|
const inline = block.source?.type === 'base64' && typeof block.source.data === 'string';
|
|
17998
|
+
// The sender's word, shown in the receipt and stored with the
|
|
17999
|
+
// file, so it's taken as a label; with nothing left of it, it
|
|
18000
|
+
// counts as absent.
|
|
18001
|
+
const claimed = block.source?.mediaType;
|
|
16653
18002
|
consider({
|
|
16654
18003
|
kind: 'attachment',
|
|
16655
18004
|
data: inline ? block.source!.data! : null,
|
|
16656
|
-
mediaType:
|
|
18005
|
+
mediaType: (typeof claimed === 'string' ? sanitizeLabel(claimed, REFERENCE_LIMITS.mimeType) : '') || 'image/png',
|
|
16657
18006
|
messagesBack,
|
|
16658
18007
|
});
|
|
16659
18008
|
} else if (block?.type === 'tool_result' && typeof block.content === 'string') {
|