@animalabs/agent-framework 0.6.2 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/framework.d.ts +58 -1
- package/dist/src/framework.d.ts.map +1 -1
- package/dist/src/framework.js +247 -19
- package/dist/src/framework.js.map +1 -1
- package/dist/src/types/framework.d.ts +40 -0
- package/dist/src/types/framework.d.ts.map +1 -1
- package/dist/src/types/trace.d.ts +9 -0
- package/dist/src/types/trace.d.ts.map +1 -1
- package/dist/test/inference-failure-observability.test.js +5 -0
- package/dist/test/inference-failure-observability.test.js.map +1 -1
- package/dist/test/maintenance-timer.test.d.ts +2 -0
- package/dist/test/maintenance-timer.test.d.ts.map +1 -0
- package/dist/test/maintenance-timer.test.js +114 -0
- package/dist/test/maintenance-timer.test.js.map +1 -0
- package/dist/test/ops-alert.test.d.ts +2 -0
- package/dist/test/ops-alert.test.d.ts.map +1 -0
- package/dist/test/ops-alert.test.js +152 -0
- package/dist/test/ops-alert.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/framework.ts +289 -19
- package/src/types/framework.ts +44 -0
- package/src/types/trace.ts +16 -0
package/dist/src/framework.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { JsStore } from '@animalabs/chronicle';
|
|
2
2
|
import type { Membrane, NormalizedRequest } from '@animalabs/membrane';
|
|
3
3
|
import { ContextManager } from '@animalabs/context-manager';
|
|
4
|
-
import type { FrameworkConfig, TraceEventListener, InferenceLogQuery, InferenceLogQueryResult, InferenceLogEntryWithId, ProcessLogQuery, ProcessLogQueryResult, ProcessLogEntryWithId, ProcessEvent, ToolCall, ToolResult, AgentConfig, Module } from './types/index.js';
|
|
4
|
+
import type { FrameworkConfig, TraceEventListener, InferenceLogQuery, InferenceLogQueryResult, InferenceLogEntryWithId, ProcessLogQuery, ProcessLogQueryResult, ProcessLogEntryWithId, ProcessEvent, ToolCall, ToolResult, AgentConfig, Module, ContextMaintenanceSnapshot } from './types/index.js';
|
|
5
5
|
import { Agent } from './agent.js';
|
|
6
6
|
import { ChannelRegistry } from './mcpl/channel-registry.js';
|
|
7
7
|
import { ConversationRouter } from './mcpl/conversation-router.js';
|
|
@@ -31,6 +31,12 @@ export declare class AgentFramework {
|
|
|
31
31
|
private traceListeners;
|
|
32
32
|
private syncIntervalMs;
|
|
33
33
|
private syncTimer;
|
|
34
|
+
private maintenanceIntervalMs;
|
|
35
|
+
private maintenanceTimer;
|
|
36
|
+
private maintenancePass;
|
|
37
|
+
private maintenanceRunId;
|
|
38
|
+
private currentMaintenanceRun;
|
|
39
|
+
private maintenanceHistory;
|
|
34
40
|
/** Last time we console-warned about stale (busy-requeued) inference requests, per agent. */
|
|
35
41
|
private staleWarnAt;
|
|
36
42
|
/** Per-agent last inference activity (epoch ms), for /healthz + doctor tooling. */
|
|
@@ -45,6 +51,19 @@ export declare class AgentFramework {
|
|
|
45
51
|
/** N consecutive failed inferences ⇒ the agent is treated as hard-down and
|
|
46
52
|
* escalated loudly to stderr. */
|
|
47
53
|
private readonly inferenceFailureEscalationThreshold;
|
|
54
|
+
/** Per-(agent,kind) timestamp of the last ops webhook post. Throttles
|
|
55
|
+
* opsAlert() so a persistent failure re-posts once per cooldown window
|
|
56
|
+
* instead of on every occurrence. */
|
|
57
|
+
private opsAlertLastSent;
|
|
58
|
+
/** Cooldown between webhook posts for the same (agent, kind). */
|
|
59
|
+
private readonly opsAlertCooldownMs;
|
|
60
|
+
/** Per-agent refusal bookkeeping for observability — exposed via
|
|
61
|
+
* healthSnapshot() and mirrored to failures.log / ops alerts. */
|
|
62
|
+
private refusalStats;
|
|
63
|
+
/** Per-agent count of consecutive refusals (reset on any non-refusal
|
|
64
|
+
* completion). Distinct from refusalRewinds, which budgets the auto-rewind
|
|
65
|
+
* loop — this one drives ops alerting. */
|
|
66
|
+
private refusalStreak;
|
|
48
67
|
/** Per-agent count of consecutive refusal-driven rewinds in the current turn
|
|
49
68
|
* chain (reset when a turn completes without a refusal). Bounds the auto
|
|
50
69
|
* rewind loop — see refusalHandling + rewindTriggeringTurn. */
|
|
@@ -121,6 +140,17 @@ export declare class AgentFramework {
|
|
|
121
140
|
* Stop the event loop.
|
|
122
141
|
*/
|
|
123
142
|
stop(): Promise<void>;
|
|
143
|
+
/**
|
|
144
|
+
* Advance queued ContextManager work without requiring a new message.
|
|
145
|
+
*
|
|
146
|
+
* Tool definitions are refreshed first because compression replays stored
|
|
147
|
+
* tool cycles and providers reject those requests when the corresponding
|
|
148
|
+
* schemas are absent. Passes never overlap; each agent gets a bounded drain
|
|
149
|
+
* so maintenance cannot monopolize the framework event loop.
|
|
150
|
+
*/
|
|
151
|
+
private startQueuedMaintenance;
|
|
152
|
+
private runQueuedMaintenance;
|
|
153
|
+
private contextProgress;
|
|
124
154
|
/**
|
|
125
155
|
* Push a process event to the queue.
|
|
126
156
|
*/
|
|
@@ -173,6 +203,8 @@ export declare class AgentFramework {
|
|
|
173
203
|
* Get all available tools from all modules and MCPL servers.
|
|
174
204
|
*/
|
|
175
205
|
getAllTools(): import('./types/index.js').ToolDefinition[];
|
|
206
|
+
/** Counts-only context-maintenance diagnostics for authenticated debug UIs. */
|
|
207
|
+
getContextMaintenanceSnapshot(): ContextMaintenanceSnapshot;
|
|
176
208
|
/**
|
|
177
209
|
* Build the membrane-normalized request that WOULD be emitted if `agentName`
|
|
178
210
|
* were activated right now — WITHOUT running inference, opening a stream, or
|
|
@@ -582,6 +614,31 @@ export declare class AgentFramework {
|
|
|
582
614
|
* work, per-agent status + last inference activity. Cheap and read-only.
|
|
583
615
|
*/
|
|
584
616
|
healthSnapshot(): Record<string, unknown>;
|
|
617
|
+
/** Append a structured JSONL record to logs/failures.log (best-effort).
|
|
618
|
+
* Durable and independent of journald/unit log redirects — this is what
|
|
619
|
+
* connectome-doctor and fleet tooling read. Always stamps `at`. Legacy
|
|
620
|
+
* records carry {at, agent, consecutive, reason}; new records add `kind`
|
|
621
|
+
* and kind-specific fields (additive only — doctor parses by regex). */
|
|
622
|
+
private logFailure;
|
|
623
|
+
/**
|
|
624
|
+
* Ops alert — the one escalation path for "a human should hear about this":
|
|
625
|
+
* (1) durable failures.log record (unless the caller already wrote one),
|
|
626
|
+
* (2) an `ops:alert` trace so authorized observers get it on the wire,
|
|
627
|
+
* (3) if CONNECTOME_OPS_WEBHOOK is set, a Discord post throttled to one per
|
|
628
|
+
* (agent, kind) per cooldown window — a persistent failure re-posts
|
|
629
|
+
* every ~15 min instead of flooding the channel on every occurrence.
|
|
630
|
+
* See connectome docs/observability.md.
|
|
631
|
+
*/
|
|
632
|
+
private opsAlert;
|
|
633
|
+
/**
|
|
634
|
+
* Record a model refusal for observability: per-agent stats + a
|
|
635
|
+
* consecutive-refusal streak (exposed via healthSnapshot), a durable
|
|
636
|
+
* failures.log record for EVERY refusal (previously invisible outside
|
|
637
|
+
* stderr), and a throttled ops alert once refusals repeat — one refusal is
|
|
638
|
+
* often survivable (auto-rewind may clear it); two in a row means stuck.
|
|
639
|
+
* The streak is reset by any non-refusal completion in the stream driver.
|
|
640
|
+
*/
|
|
641
|
+
private noteRefusal;
|
|
585
642
|
private noteInferenceExhausted;
|
|
586
643
|
/**
|
|
587
644
|
* Automatic poison-history quarantine (the actual "breaker"). Sheds the
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"framework.d.ts","sourceRoot":"","sources":["../../src/framework.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,KAAK,EAAE,QAAQ,EAAgB,iBAAiB,EAA4E,MAAM,qBAAqB,CAAC;AAE/J,OAAO,EAAE,cAAc,EAAuB,MAAM,4BAA4B,CAAC;AAQjF,OAAO,KAAK,EACV,eAAe,EAMf,kBAAkB,EAElB,iBAAiB,EACjB,uBAAuB,EACvB,uBAAuB,EAGvB,eAAe,EACf,qBAAqB,EACrB,qBAAqB,EAErB,YAAY,EAGZ,QAAQ,EAER,UAAU,EACV,WAAW,EAGX,MAAM,
|
|
1
|
+
{"version":3,"file":"framework.d.ts","sourceRoot":"","sources":["../../src/framework.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,KAAK,EAAE,QAAQ,EAAgB,iBAAiB,EAA4E,MAAM,qBAAqB,CAAC;AAE/J,OAAO,EAAE,cAAc,EAAuB,MAAM,4BAA4B,CAAC;AAQjF,OAAO,KAAK,EACV,eAAe,EAMf,kBAAkB,EAElB,iBAAiB,EACjB,uBAAuB,EACvB,uBAAuB,EAGvB,eAAe,EACf,qBAAqB,EACrB,qBAAqB,EAErB,YAAY,EAGZ,QAAQ,EAER,UAAU,EACV,WAAW,EAGX,MAAM,EAIN,0BAA0B,EAC3B,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAQnC,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAQnE,OAAO,KAAK,EAAE,oBAAoB,EAAqB,MAAM,kBAAkB,CAAC;AA6KhF,qBAAa,cAAc;IACzB,OAAO,CAAC,KAAK,CAAU;IACvB,OAAO,CAAC,SAAS,CAAU;IAC3B,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,KAAK,CAAmB;IAChC,OAAO,CAAC,MAAM,CAAiC;IAC/C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,eAAe,CAA0B;IACjD;;;;;;;;OAQG;IACH,OAAO,CAAC,qBAAqB,CAAkC;IAC/D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,WAAW,CAA8B;IACjD,OAAO,CAAC,cAAc,CAA4B;IAClD,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAA+C;IAChE,OAAO,CAAC,qBAAqB,CAAS;IACtC,OAAO,CAAC,gBAAgB,CAA+C;IACvE,OAAO,CAAC,eAAe,CAA8B;IACrD,OAAO,CAAC,gBAAgB,CAAK;IAC7B,OAAO,CAAC,qBAAqB,CAAsC;IACnE,OAAO,CAAC,kBAAkB,CAA+B;IACzD,6FAA6F;IAC7F,OAAO,CAAC,WAAW,CAA6B;IAChD,mFAAmF;IACnF,OAAO,CAAC,eAAe,CAAsG;IAC7H,OAAO,CAAC,qBAAqB,CAAU;IACvC,OAAO,CAAC,uBAAuB,CAAU;IACzC,OAAO,CAAC,aAAa,CAAyC;IAC9D,OAAO,CAAC,sBAAsB,CAA0C;IACxE;oEACgE;IAChE,OAAO,CAAC,4BAA4B,CAAkC;IACtE;sCACkC;IAClC,OAAO,CAAC,QAAQ,CAAC,mCAAmC,CAAK;IACzD;;0CAEsC;IACtC,OAAO,CAAC,gBAAgB,CAAkC;IAC1D,iEAAiE;IACjE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAe;IAClD;sEACkE;IAClE,OAAO,CAAC,YAAY,CAAuH;IAC3I;;+CAE2C;IAC3C,OAAO,CAAC,aAAa,CAAkC;IACvD;;oEAEgE;IAChE,OAAO,CAAC,cAAc,CAAkC;IACxD;;;6EAGyE;IACzE,OAAO,CAAC,iBAAiB,CAAkC;IAC3D;;yEAEqE;IACrE,OAAO,CAAC,aAAa,CAAoF;IACzG;;iFAE6E;IAC7E,OAAO,CAAC,YAAY,CAKL;IACf,gFAAgF;IAChF,OAAO,CAAC,gBAAgB,CAAuB;IAI/C,OAAO,CAAC,gBAAgB,CAA2F;IAGnH,OAAO,CAAC,YAAY,CAAkC;IACtD,OAAO,CAAC,UAAU,CAAuC;IAEzD,kFAAkF;IAClF,OAAO,CAAC,gBAAgB,CAA0E;IAGlG,OAAO,CAAC,kBAAkB,CAAmC;IAC7D,OAAO,CAAC,iBAAiB,CAAkC;IAC3D,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,gBAAgB,CAAiC;IACzD,OAAO,CAAC,WAAW,CAA4B;IAC/C,OAAO,CAAC,eAAe,CAAgC;IACvD,OAAO,CAAC,eAAe,CAAgC;IACvD,OAAO,CAAC,iBAAiB,CAAkC;IAC3D,+EAA+E;IAC/E,OAAO,CAAC,kBAAkB,CAAmC;IAC7D,gFAAgF;IAChF,OAAO,CAAC,YAAY,CAAuC;IAC3D;;;;OAIG;IACH,OAAO,CAAC,sBAAsB,CAAkC;IAChE,qCAAqC;IACrC,OAAO,CAAC,qBAAqB,CAAK;IAElC;6EACyE;IACzE,OAAO,CAAC,yBAAyB,CAA0B;IAC3D,OAAO,CAAC,SAAS,CAAmD;IACpE,OAAO,CAAC,uBAAuB,CAAS;IACxC,OAAO,CAAC,sBAAsB,CAAS;IACvC,wDAAwD;IACxD,OAAO,CAAC,aAAa,CAAkC;IACvD,0DAA0D;IAC1D,OAAO,CAAC,iBAAiB,CAAsE;IAC/F;gFAC4E;IAC5E,OAAO,CAAC,oBAAoB,CAAqC;IACjE;;0DAEsD;IACtD,OAAO,CAAC,0BAA0B,CAAiE;IAGnG,OAAO,CAAC,SAAS,CAA0B;IAG3C,OAAO,CAAC,YAAY,CAAe;IAEnC,OAAO;IAuCP;;OAEG;WACU,MAAM,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC;IAyMrE;;OAEG;IACH,KAAK,IAAI,IAAI;IA8Bb;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IA6D3B;;;;;;;OAOG;IACH,OAAO,CAAC,sBAAsB;YAchB,oBAAoB;IAgElC,OAAO,CAAC,eAAe;IAWvB;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI;IAKpC;;OAEG;IACH,OAAO,CAAC,QAAQ,EAAE,kBAAkB,GAAG,MAAM,IAAI;IAWjD;;;;;OAKG;IACH,IAAI,QAAQ,IAAI,eAAe,GAAG,IAAI,CAErC;IAED;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,kBAAkB,GAAG,IAAI;IAO5C;;OAEG;IACG,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAK9C;;OAEG;IACG,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAK/C;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,GAAG,IAAI;IAIpC;;OAEG;IACH,YAAY,IAAI,KAAK,EAAE;IAIvB;;;OAGG;IACH,qBAAqB,IAAI,kBAAkB,GAAG,IAAI;IAIlD;;OAEG;IACH,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO;IAY3D;;OAEG;IACH,aAAa,IAAI,MAAM,EAAE;IAIzB;;OAEG;IACH,WAAW,IAAI,OAAO,kBAAkB,EAAE,cAAc,EAAE;IAiB1D,+EAA+E;IAC/E,6BAA6B,IAAI,0BAA0B;IAqB3D;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACG,iBAAiB,CACrB,SAAS,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,OAAO,CAAA;KAAE,GAC9B,OAAO,CAAC,iBAAiB,CAAC;IAiD7B;;OAEG;IACH,uBAAuB,IAAI;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,SAAS,EAAE,OAAO,CAAA;KAAE;IAOnE;;OAEG;IACH,QAAQ,IAAI,OAAO;IAInB;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAWtC;;;;;OAKG;IACH,aAAa,CACX,SAAS,EAAE,OAAO,EAClB,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAA;KAAE,GAC5C,OAAO,iBAAiB,EAAE,UAAU;IAOvC;;;OAGG;IACH,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAOvC,6EAA6E;IAC7E,kBAAkB,IAAI,MAAM,EAAE;IAI9B;;OAEG;IACH,WAAW,IAAI,QAAQ;IAIvB,eAAe,IAAI,oBAAoB;IAIvC,OAAO,CAAC,iBAAiB;IAezB,OAAO,CAAC,iBAAiB;IAWzB;;;;;;;;;;;OAWG;IACG,oBAAoB,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC;QACvD,KAAK,EAAE,KAAK,CAAC;QACb,cAAc,EAAE,cAAc,CAAC;QAC/B,OAAO,EAAE,MAAM,IAAI,CAAC;KACrB,CAAC;IAsBF;;;;;;;;;OASG;IACG,wBAAwB,CAC5B,KAAK,EAAE,KAAK,EACZ,cAAc,EAAE,cAAc,GAC7B,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAA;KAAE,CAAC;IA+ItD;;OAEG;IACH,aAAa,IAAI,MAAM;IAIvB;;;OAGG;IACH,8EAA8E;IAC9E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CA8BjC;IAEF;;;4CAGwC;IACxC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAqGrC;IAEF;6EACyE;IACzE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CASrC;IAEF;+DAC2D;IAC3D,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAwBpC;IAEF,4EAA4E;IAC5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAMvC;IAEF;;;;;OAKG;YACW,cAAc;IAoC5B;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,cAAc;IAkEtB;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;IAiC1B;;;;OAIG;YACW,cAAc;IA6B5B;;;;OAIG;IACH,OAAO,CAAC,aAAa;IAkBrB,yEAAyE;YAC3D,aAAa;IAU3B;;;;;;;;OAQG;YACW,iBAAiB;IA+M/B;;;;OAIG;YACW,kBAAkB;IAsChC,kBAAkB,CAAC,KAAK,CAAC,EAAE,iBAAiB,GAAG,uBAAuB;IAuEtE;;;OAGG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,UAAO,GAAG,uBAAuB,GAAG,IAAI;IAuCtF;;OAEG;IACH,iBAAiB,CAAC,KAAK,SAAK,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,uBAAuB,EAAE;IAQ5E;;;OAGG;IACH,gBAAgB,CAAC,KAAK,CAAC,EAAE,eAAe,GAAG,qBAAqB;IAwFhE;;;OAGG;IACH,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,UAAO,GAAG,qBAAqB,GAAG,IAAI;IAuBlF;;OAEG;IACH,eAAe,CAAC,KAAK,SAAK,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,qBAAqB,EAAE;IAYxE;;;;;;;;;OASG;IACH,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG;QAC/B,MAAM,EAAE,OAAO,CAAC;QAChB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB;IAqDD;;;;;OAKG;IACH,IAAI,CAAC,SAAS,EAAE,MAAM,GAAG;QACvB,MAAM,EAAE,OAAO,CAAC;QAChB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB;IA4CD;;OAEG;IACH,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG;QACnC,OAAO,EAAE,OAAO,CAAC;QACjB,OAAO,EAAE,OAAO,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,MAAM,CAAC;KACnB;IAeD,OAAO,CAAC,oBAAoB;IAuB5B;;;;;;;;;;;;;;;;;OAiBG;IACH,OAAO,CAAC,oBAAoB,CAA4B;IAExD,OAAO,CAAC,oBAAoB;IAU5B,OAAO,CAAC,kBAAkB;IAoB1B,OAAO,CAAC,mBAAmB;IAU3B;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAsB5B;;;OAGG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;YAarB,WAAW;YAsBX,OAAO;YAUP,gBAAgB;YAwBhB,kBAAkB;YAiLlB,oBAAoB;IAwElC;;;OAGG;YACW,yBAAyB;IAgDvC;;;;;OAKG;YACW,yBAAyB;IA0FvC;;;;;;OAMG;YACW,uBAAuB;IAyCrC,4EAA4E;IAC5E,OAAO,CAAC,8BAA8B;IAWtC;;;;;OAKG;IACH,OAAO,CAAC,wBAAwB;IAchC;;;;;OAKG;IACH,OAAO,CAAC,uBAAuB;IAkB/B;;;;;;OAMG;IACH,OAAO,CAAC,yBAAyB;IAoDjC;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA0C3B;;;;;;;;;;;;;;;OAeG;IACH,OAAO,CAAC,sBAAsB;YAsBhB,wBAAwB;YA4FxB,gBAAgB;YAuGhB,WAAW;IA0oBzB;;;;OAIG;IACG,eAAe,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC;YAkB5C,mBAAmB;IAgCjC,OAAO,CAAC,oBAAoB;IAuB5B;;;;;;OAMG;IACH,OAAO,CAAC,0BAA0B;IA2BlC,OAAO,CAAC,qBAAqB;IAO7B,OAAO,CAAC,YAAY;IA8BpB,OAAO,CAAC,eAAe;IAqBvB;;;OAGG;IACH,OAAO,CAAC,eAAe;IASvB,OAAO,CAAC,gBAAgB;IAiExB,OAAO,CAAC,qBAAqB;IAyD7B,OAAO,CAAC,UAAU;IAwBlB,OAAO,CAAC,WAAW;IAUnB,OAAO,CAAC,aAAa;IAUrB,OAAO,CAAC,UAAU;IAUlB,OAAO,CAAC,aAAa;IAUrB,OAAO,CAAC,oBAAoB;IAgB5B,OAAO,CAAC,SAAS;IA0CjB;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH;;;OAGG;IACH,cAAc,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAiBzC;;;;6EAIyE;IACzE,OAAO,CAAC,UAAU;IAUlB;;;;;;;;OAQG;IACH,OAAO,CAAC,QAAQ;IA8BhB;;;;;;;OAOG;IACH,OAAO,CAAC,WAAW;IAmCnB,OAAO,CAAC,sBAAsB;IAwE9B;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,yBAAyB;IAyCjC;;;OAGG;YACW,cAAc;IA0I5B;;;;;OAKG;YACW,yBAAyB;IAgCvC;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,0BAA0B;IA6ClC;;;;;OAKG;IACG,iBAAiB,CACrB,MAAM,EAAE,OAAO,iBAAiB,EAAE,gBAAgB,GACjD,OAAO,CAAC,IAAI,CAAC;IA4BhB;;;;;;OAMG;IACG,oBAAoB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAyBrD;;;;;OAKG;IACG,iBAAiB,CACrB,EAAE,EAAE,MAAM,EACV,SAAS,CAAC,EAAE,OAAO,iBAAiB,EAAE,gBAAgB,GACrD,OAAO,CAAC,IAAI,CAAC;IAShB;;;OAGG;IACH,eAAe,IAAI,KAAK,CAAC;QACvB,EAAE,EAAE,MAAM,CAAC;QACX,SAAS,EAAE,OAAO,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,GAAG,CAAC,EAAE,MAAM,CAAC;KACd,CAAC;IAoBF;;;OAGG;IACH,OAAO,CAAC,cAAc;IA6MtB;;;OAGG;YACW,gBAAgB;IA4B9B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAyB9B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAsBxB;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAwH5B;;OAEG;IACH;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAkE/B;;;;;OAKG;IACH,OAAO,CAAC,wBAAwB;IA+EhC;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IA0C7B;2DACuD;IACvD,OAAO,CAAC,oBAAoB;IAoB5B,OAAO,CAAC,yBAAyB;IAcjC,OAAO,CAAC,oBAAoB;IA6B5B,OAAO,CAAC,0BAA0B;CAsBnC"}
|
package/dist/src/framework.js
CHANGED
|
@@ -66,6 +66,10 @@ class DefaultErrorPolicy {
|
|
|
66
66
|
}
|
|
67
67
|
/** Default sync interval in milliseconds */
|
|
68
68
|
const DEFAULT_SYNC_INTERVAL_MS = 1000;
|
|
69
|
+
/** Poll pending context-strategy maintenance independently of user activity. */
|
|
70
|
+
const DEFAULT_MAINTENANCE_INTERVAL_MS = 5000;
|
|
71
|
+
/** Bound one pass so a large backlog yields to inference and other agents. */
|
|
72
|
+
const MAINTENANCE_TICKS_PER_PASS = 8;
|
|
69
73
|
/**
|
|
70
74
|
* Extract fields the EventGate cares about from a ProcessEvent. The set of
|
|
71
75
|
* event variants that carry `content`/`mount`/`paths`/`metadata` is open-ended
|
|
@@ -128,6 +132,12 @@ export class AgentFramework {
|
|
|
128
132
|
traceListeners = [];
|
|
129
133
|
syncIntervalMs;
|
|
130
134
|
syncTimer = null;
|
|
135
|
+
maintenanceIntervalMs;
|
|
136
|
+
maintenanceTimer = null;
|
|
137
|
+
maintenancePass = null;
|
|
138
|
+
maintenanceRunId = 0;
|
|
139
|
+
currentMaintenanceRun = null;
|
|
140
|
+
maintenanceHistory = [];
|
|
131
141
|
/** Last time we console-warned about stale (busy-requeued) inference requests, per agent. */
|
|
132
142
|
staleWarnAt = new Map();
|
|
133
143
|
/** Per-agent last inference activity (epoch ms), for /healthz + doctor tooling. */
|
|
@@ -142,6 +152,19 @@ export class AgentFramework {
|
|
|
142
152
|
/** N consecutive failed inferences ⇒ the agent is treated as hard-down and
|
|
143
153
|
* escalated loudly to stderr. */
|
|
144
154
|
inferenceFailureEscalationThreshold = 3;
|
|
155
|
+
/** Per-(agent,kind) timestamp of the last ops webhook post. Throttles
|
|
156
|
+
* opsAlert() so a persistent failure re-posts once per cooldown window
|
|
157
|
+
* instead of on every occurrence. */
|
|
158
|
+
opsAlertLastSent = new Map();
|
|
159
|
+
/** Cooldown between webhook posts for the same (agent, kind). */
|
|
160
|
+
opsAlertCooldownMs = 15 * 60_000;
|
|
161
|
+
/** Per-agent refusal bookkeeping for observability — exposed via
|
|
162
|
+
* healthSnapshot() and mirrored to failures.log / ops alerts. */
|
|
163
|
+
refusalStats = new Map();
|
|
164
|
+
/** Per-agent count of consecutive refusals (reset on any non-refusal
|
|
165
|
+
* completion). Distinct from refusalRewinds, which budgets the auto-rewind
|
|
166
|
+
* loop — this one drives ops alerting. */
|
|
167
|
+
refusalStreak = new Map();
|
|
145
168
|
/** Per-agent count of consecutive refusal-driven rewinds in the current turn
|
|
146
169
|
* chain (reset when a turn completes without a refusal). Bounds the auto
|
|
147
170
|
* rewind loop — see refusalHandling + rewindTriggeringTurn. */
|
|
@@ -211,13 +234,14 @@ export class AgentFramework {
|
|
|
211
234
|
eventGate = null;
|
|
212
235
|
// Session-level token usage tracking (always-on)
|
|
213
236
|
usageTracker;
|
|
214
|
-
constructor(store, ownsStore, membrane, inferencePolicy, errorPolicy, syncIntervalMs, processLoggingPersist, processLoggingBroadcast) {
|
|
237
|
+
constructor(store, ownsStore, membrane, inferencePolicy, errorPolicy, syncIntervalMs, maintenanceIntervalMs, processLoggingPersist, processLoggingBroadcast) {
|
|
215
238
|
this.store = store;
|
|
216
239
|
this.ownsStore = ownsStore;
|
|
217
240
|
this.membrane = membrane;
|
|
218
241
|
this.inferencePolicy = inferencePolicy;
|
|
219
242
|
this.errorPolicy = errorPolicy;
|
|
220
243
|
this.syncIntervalMs = syncIntervalMs;
|
|
244
|
+
this.maintenanceIntervalMs = maintenanceIntervalMs;
|
|
221
245
|
this.processLoggingPersist = processLoggingPersist;
|
|
222
246
|
this.processLoggingBroadcast = processLoggingBroadcast;
|
|
223
247
|
this.queue = new ProcessQueueImpl();
|
|
@@ -299,7 +323,7 @@ export class AgentFramework {
|
|
|
299
323
|
// Already registered
|
|
300
324
|
}
|
|
301
325
|
}
|
|
302
|
-
const framework = new AgentFramework(store, ownsStore, config.membrane, config.inferencePolicy ?? new DefaultInferencePolicy(), config.errorPolicy ?? new DefaultErrorPolicy(), config.syncIntervalMs ?? DEFAULT_SYNC_INTERVAL_MS, processLoggingPersist, processLoggingBroadcast);
|
|
326
|
+
const framework = new AgentFramework(store, ownsStore, config.membrane, config.inferencePolicy ?? new DefaultInferencePolicy(), config.errorPolicy ?? new DefaultErrorPolicy(), config.syncIntervalMs ?? DEFAULT_SYNC_INTERVAL_MS, config.maintenanceIntervalMs ?? DEFAULT_MAINTENANCE_INTERVAL_MS, processLoggingPersist, processLoggingBroadcast);
|
|
303
327
|
// Restore persisted usage data (if any) from prior session
|
|
304
328
|
framework.restoreUsageState();
|
|
305
329
|
// Create agents
|
|
@@ -433,6 +457,15 @@ export class AgentFramework {
|
|
|
433
457
|
}
|
|
434
458
|
}, this.syncIntervalMs);
|
|
435
459
|
}
|
|
460
|
+
if (this.maintenanceIntervalMs > 0) {
|
|
461
|
+
this.maintenanceTimer = setInterval(() => {
|
|
462
|
+
this.startQueuedMaintenance();
|
|
463
|
+
}, this.maintenanceIntervalMs);
|
|
464
|
+
this.maintenanceTimer.unref?.();
|
|
465
|
+
// Do not make a restored queue wait a full interval before its first
|
|
466
|
+
// attempt. MCPL/module initialization has completed before start().
|
|
467
|
+
this.startQueuedMaintenance();
|
|
468
|
+
}
|
|
436
469
|
}
|
|
437
470
|
/**
|
|
438
471
|
* Stop the event loop.
|
|
@@ -456,6 +489,13 @@ export class AgentFramework {
|
|
|
456
489
|
clearInterval(this.syncTimer);
|
|
457
490
|
this.syncTimer = null;
|
|
458
491
|
}
|
|
492
|
+
if (this.maintenanceTimer) {
|
|
493
|
+
clearInterval(this.maintenanceTimer);
|
|
494
|
+
this.maintenanceTimer = null;
|
|
495
|
+
}
|
|
496
|
+
if (this.maintenancePass) {
|
|
497
|
+
await this.maintenancePass;
|
|
498
|
+
}
|
|
459
499
|
if (this.loopPromise) {
|
|
460
500
|
await this.loopPromise;
|
|
461
501
|
}
|
|
@@ -481,6 +521,100 @@ export class AgentFramework {
|
|
|
481
521
|
this.store.close();
|
|
482
522
|
}
|
|
483
523
|
}
|
|
524
|
+
/**
|
|
525
|
+
* Advance queued ContextManager work without requiring a new message.
|
|
526
|
+
*
|
|
527
|
+
* Tool definitions are refreshed first because compression replays stored
|
|
528
|
+
* tool cycles and providers reject those requests when the corresponding
|
|
529
|
+
* schemas are absent. Passes never overlap; each agent gets a bounded drain
|
|
530
|
+
* so maintenance cannot monopolize the framework event loop.
|
|
531
|
+
*/
|
|
532
|
+
startQueuedMaintenance() {
|
|
533
|
+
if (this.maintenancePass || !this.running)
|
|
534
|
+
return;
|
|
535
|
+
const pass = this.runQueuedMaintenance().catch((error) => {
|
|
536
|
+
console.error('[context-maintenance] pass failed:', error instanceof Error ? error.message : error);
|
|
537
|
+
});
|
|
538
|
+
this.maintenancePass = pass;
|
|
539
|
+
void pass.finally(() => {
|
|
540
|
+
if (this.maintenancePass === pass)
|
|
541
|
+
this.maintenancePass = null;
|
|
542
|
+
});
|
|
543
|
+
}
|
|
544
|
+
async runQueuedMaintenance() {
|
|
545
|
+
const allTools = this.getAllTools();
|
|
546
|
+
const queued = [...this.agents.values()].flatMap((agent) => {
|
|
547
|
+
const cm = agent.getContextManager();
|
|
548
|
+
const tools = allTools.filter((tool) => agent.canUseTool(tool.name));
|
|
549
|
+
cm.setToolDefinitions(tools);
|
|
550
|
+
if (cm.isReady())
|
|
551
|
+
return [];
|
|
552
|
+
const pending = cm.getPendingWork()?.description;
|
|
553
|
+
const progress = this.contextProgress(cm);
|
|
554
|
+
const record = {
|
|
555
|
+
agentName: agent.name,
|
|
556
|
+
startedAt: Date.now(),
|
|
557
|
+
ticks: 0,
|
|
558
|
+
readyBefore: false,
|
|
559
|
+
...(pending ? { pendingBefore: pending } : {}),
|
|
560
|
+
...(progress ? { progressBefore: progress } : {}),
|
|
561
|
+
};
|
|
562
|
+
return [{
|
|
563
|
+
agent,
|
|
564
|
+
cm,
|
|
565
|
+
record,
|
|
566
|
+
}];
|
|
567
|
+
});
|
|
568
|
+
if (queued.length === 0)
|
|
569
|
+
return;
|
|
570
|
+
const run = {
|
|
571
|
+
id: ++this.maintenanceRunId,
|
|
572
|
+
startedAt: Date.now(),
|
|
573
|
+
agents: queued.map(item => item.record),
|
|
574
|
+
};
|
|
575
|
+
this.currentMaintenanceRun = run;
|
|
576
|
+
try {
|
|
577
|
+
await Promise.all(queued.map(async ({ agent, cm, record }) => {
|
|
578
|
+
try {
|
|
579
|
+
for (let i = 0; i < MAINTENANCE_TICKS_PER_PASS && this.running && !cm.isReady(); i++) {
|
|
580
|
+
await cm.tick();
|
|
581
|
+
record.ticks++;
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
catch (error) {
|
|
585
|
+
record.error = error instanceof Error ? error.message : String(error);
|
|
586
|
+
console.error(`[context-maintenance] agent=${agent.name} failed:`, record.error);
|
|
587
|
+
}
|
|
588
|
+
finally {
|
|
589
|
+
const pending = cm.getPendingWork()?.description;
|
|
590
|
+
record.finishedAt = Date.now();
|
|
591
|
+
record.readyAfter = cm.isReady();
|
|
592
|
+
if (pending)
|
|
593
|
+
record.pendingAfter = pending;
|
|
594
|
+
const progress = this.contextProgress(cm);
|
|
595
|
+
if (progress)
|
|
596
|
+
record.progressAfter = progress;
|
|
597
|
+
}
|
|
598
|
+
}));
|
|
599
|
+
}
|
|
600
|
+
finally {
|
|
601
|
+
run.finishedAt = Date.now();
|
|
602
|
+
run.agents.sort((a, b) => a.agentName.localeCompare(b.agentName));
|
|
603
|
+
this.currentMaintenanceRun = null;
|
|
604
|
+
this.maintenanceHistory.unshift(run);
|
|
605
|
+
if (this.maintenanceHistory.length > 50)
|
|
606
|
+
this.maintenanceHistory.length = 50;
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
contextProgress(cm) {
|
|
610
|
+
const strategy = cm.getStrategy();
|
|
611
|
+
if (typeof strategy.getProgressSnapshot !== 'function')
|
|
612
|
+
return undefined;
|
|
613
|
+
const progress = strategy.getProgressSnapshot();
|
|
614
|
+
return progress && typeof progress === 'object'
|
|
615
|
+
? progress
|
|
616
|
+
: undefined;
|
|
617
|
+
}
|
|
484
618
|
/**
|
|
485
619
|
* Push a process event to the queue.
|
|
486
620
|
*/
|
|
@@ -592,6 +726,27 @@ export class AgentFramework {
|
|
|
592
726
|
}
|
|
593
727
|
return [...moduleTools, ...this.mcplTools, ...channelTools, ...gateTools];
|
|
594
728
|
}
|
|
729
|
+
/** Counts-only context-maintenance diagnostics for authenticated debug UIs. */
|
|
730
|
+
getContextMaintenanceSnapshot() {
|
|
731
|
+
const agents = [...this.agents.values()].map((agent) => {
|
|
732
|
+
const cm = agent.getContextManager();
|
|
733
|
+
const pending = cm.getPendingWork()?.description;
|
|
734
|
+
const progress = this.contextProgress(cm);
|
|
735
|
+
return {
|
|
736
|
+
agentName: agent.name,
|
|
737
|
+
ready: cm.isReady(),
|
|
738
|
+
...(pending ? { pending } : {}),
|
|
739
|
+
...(progress ? { progress } : {}),
|
|
740
|
+
};
|
|
741
|
+
});
|
|
742
|
+
return structuredClone({
|
|
743
|
+
intervalMs: this.maintenanceIntervalMs,
|
|
744
|
+
ticksPerPass: MAINTENANCE_TICKS_PER_PASS,
|
|
745
|
+
current: this.currentMaintenanceRun,
|
|
746
|
+
history: this.maintenanceHistory,
|
|
747
|
+
agents,
|
|
748
|
+
});
|
|
749
|
+
}
|
|
595
750
|
/**
|
|
596
751
|
* Build the membrane-normalized request that WOULD be emitted if `agentName`
|
|
597
752
|
* were activated right now — WITHOUT running inference, opening a stream, or
|
|
@@ -2970,6 +3125,7 @@ export class AgentFramework {
|
|
|
2970
3125
|
const category = stopDetails?.category ?? 'unknown';
|
|
2971
3126
|
console.error(`[inference-refusal] agent=${agent.name} category=${category}` +
|
|
2972
3127
|
(stopDetails?.explanation ? ` explanation=${stopDetails.explanation}` : ''));
|
|
3128
|
+
this.noteRefusal(agent.name, category, tokenUsage);
|
|
2973
3129
|
// Rewind: excise the turn that fed the refusal, drop a
|
|
2974
3130
|
// metadata-only marker, and retry — keeping the agent on its own
|
|
2975
3131
|
// model instead of substituting a fallback. Driven either by an
|
|
@@ -3037,6 +3193,7 @@ export class AgentFramework {
|
|
|
3037
3193
|
this.refusalRewinds.set(agent.name, 0);
|
|
3038
3194
|
}
|
|
3039
3195
|
this.rewindEpisode.delete(agent.name);
|
|
3196
|
+
this.refusalStreak.delete(agent.name);
|
|
3040
3197
|
}
|
|
3041
3198
|
// Dispatch speech (and thoughts if any)
|
|
3042
3199
|
if (speechContent.length > 0 || thoughts.length > 0) {
|
|
@@ -3740,9 +3897,84 @@ export class AgentFramework {
|
|
|
3740
3897
|
status: agent.state.status,
|
|
3741
3898
|
consecutiveInferenceFailures: this.consecutiveInferenceFailures.get(name) ?? 0,
|
|
3742
3899
|
lastInference: this.lastInferenceAt.get(name) ?? null,
|
|
3900
|
+
refusalStats: this.refusalStats.get(name) ?? null,
|
|
3743
3901
|
})),
|
|
3744
3902
|
};
|
|
3745
3903
|
}
|
|
3904
|
+
/** Append a structured JSONL record to logs/failures.log (best-effort).
|
|
3905
|
+
* Durable and independent of journald/unit log redirects — this is what
|
|
3906
|
+
* connectome-doctor and fleet tooling read. Always stamps `at`. Legacy
|
|
3907
|
+
* records carry {at, agent, consecutive, reason}; new records add `kind`
|
|
3908
|
+
* and kind-specific fields (additive only — doctor parses by regex). */
|
|
3909
|
+
logFailure(record) {
|
|
3910
|
+
try {
|
|
3911
|
+
mkdirSync('logs', { recursive: true });
|
|
3912
|
+
appendFileSync('logs/failures.log', JSON.stringify({ at: new Date().toISOString(), ...record }) + '\n');
|
|
3913
|
+
}
|
|
3914
|
+
catch { /* best-effort */ }
|
|
3915
|
+
}
|
|
3916
|
+
/**
|
|
3917
|
+
* Ops alert — the one escalation path for "a human should hear about this":
|
|
3918
|
+
* (1) durable failures.log record (unless the caller already wrote one),
|
|
3919
|
+
* (2) an `ops:alert` trace so authorized observers get it on the wire,
|
|
3920
|
+
* (3) if CONNECTOME_OPS_WEBHOOK is set, a Discord post throttled to one per
|
|
3921
|
+
* (agent, kind) per cooldown window — a persistent failure re-posts
|
|
3922
|
+
* every ~15 min instead of flooding the channel on every occurrence.
|
|
3923
|
+
* See connectome docs/observability.md.
|
|
3924
|
+
*/
|
|
3925
|
+
opsAlert(kind, agentName, message, opts) {
|
|
3926
|
+
if (!opts?.skipLog) {
|
|
3927
|
+
this.logFailure({ agent: agentName, kind, reason: message, ...opts?.data });
|
|
3928
|
+
}
|
|
3929
|
+
this.emitTrace({ type: 'ops:alert', kind, agentName, message, data: opts?.data });
|
|
3930
|
+
const hook = process.env.CONNECTOME_OPS_WEBHOOK;
|
|
3931
|
+
if (!hook)
|
|
3932
|
+
return;
|
|
3933
|
+
const key = `${agentName}:${kind}`;
|
|
3934
|
+
const now = Date.now();
|
|
3935
|
+
if (now - (this.opsAlertLastSent.get(key) ?? 0) < this.opsAlertCooldownMs)
|
|
3936
|
+
return;
|
|
3937
|
+
// Stamp BEFORE the async post so a burst can't race past the cooldown.
|
|
3938
|
+
this.opsAlertLastSent.set(key, now);
|
|
3939
|
+
fetch(hook, {
|
|
3940
|
+
method: 'POST',
|
|
3941
|
+
headers: { 'content-type': 'application/json' },
|
|
3942
|
+
body: JSON.stringify({
|
|
3943
|
+
content: `\u{1F6A8} **${agentName}** ${kind}: ${message.slice(0, 500)}`,
|
|
3944
|
+
}),
|
|
3945
|
+
}).catch((err) => console.error(`[ops-alert] webhook post failed (${key}):`, err instanceof Error ? err.message : err));
|
|
3946
|
+
}
|
|
3947
|
+
/**
|
|
3948
|
+
* Record a model refusal for observability: per-agent stats + a
|
|
3949
|
+
* consecutive-refusal streak (exposed via healthSnapshot), a durable
|
|
3950
|
+
* failures.log record for EVERY refusal (previously invisible outside
|
|
3951
|
+
* stderr), and a throttled ops alert once refusals repeat — one refusal is
|
|
3952
|
+
* often survivable (auto-rewind may clear it); two in a row means stuck.
|
|
3953
|
+
* The streak is reset by any non-refusal completion in the stream driver.
|
|
3954
|
+
*/
|
|
3955
|
+
noteRefusal(agentName, category, tokens) {
|
|
3956
|
+
const stats = this.refusalStats.get(agentName)
|
|
3957
|
+
?? { total: 0, byCategory: {}, lastAt: 0, lastCategory: '' };
|
|
3958
|
+
stats.total += 1;
|
|
3959
|
+
stats.byCategory[category] = (stats.byCategory[category] ?? 0) + 1;
|
|
3960
|
+
stats.lastAt = Date.now();
|
|
3961
|
+
stats.lastCategory = category;
|
|
3962
|
+
this.refusalStats.set(agentName, stats);
|
|
3963
|
+
const streak = (this.refusalStreak.get(agentName) ?? 0) + 1;
|
|
3964
|
+
this.refusalStreak.set(agentName, streak);
|
|
3965
|
+
this.logFailure({
|
|
3966
|
+
agent: agentName,
|
|
3967
|
+
kind: 'refusal',
|
|
3968
|
+
category,
|
|
3969
|
+
streak,
|
|
3970
|
+
locus: this.channelRegistry?.buildChannelContext()?.incoming?.channelId ?? null,
|
|
3971
|
+
tokens: tokens ?? null,
|
|
3972
|
+
});
|
|
3973
|
+
if (streak >= 2) {
|
|
3974
|
+
this.opsAlert('refusal', agentName, `refusal streak ${streak}, category=${category}`, { skipLog: true, data: { category, streak } });
|
|
3975
|
+
}
|
|
3976
|
+
return streak;
|
|
3977
|
+
}
|
|
3746
3978
|
noteInferenceExhausted(agentName, reason, retryable, errorType) {
|
|
3747
3979
|
const streak = (this.consecutiveInferenceFailures.get(agentName) ?? 0) + 1;
|
|
3748
3980
|
this.consecutiveInferenceFailures.set(agentName, streak);
|
|
@@ -3751,12 +3983,8 @@ export class AgentFramework {
|
|
|
3751
3983
|
console.error(`[inference-failed] agent=${agentName} consecutive=${streak}: ${reason}`);
|
|
3752
3984
|
// (1b) Machine-greppable durable record, independent of journald/unit log
|
|
3753
3985
|
// redirects: logs/failures.log under the host's working directory. This is
|
|
3754
|
-
// what connectome-doctor reads.
|
|
3755
|
-
|
|
3756
|
-
mkdirSync('logs', { recursive: true });
|
|
3757
|
-
appendFileSync('logs/failures.log', JSON.stringify({ at: new Date().toISOString(), agent: agentName, consecutive: streak, reason }) + '\n');
|
|
3758
|
-
}
|
|
3759
|
-
catch { /* best-effort */ }
|
|
3986
|
+
// what connectome-doctor reads. Legacy fields kept; `kind` is additive.
|
|
3987
|
+
this.logFailure({ agent: agentName, consecutive: streak, reason, kind: 'inference-exhausted' });
|
|
3760
3988
|
// (2) Agent-facing chronicle marker (no inference triggered → no loop).
|
|
3761
3989
|
const agent = this.agents.get(agentName);
|
|
3762
3990
|
if (agent && process.env.SUPPRESS_INFERENCE_FAILED_MARKER !== '1') {
|
|
@@ -3778,17 +4006,10 @@ export class AgentFramework {
|
|
|
3778
4006
|
if (streak >= this.inferenceFailureEscalationThreshold) {
|
|
3779
4007
|
console.error(`[inference-hard-down] agent=${agentName} has FAILED ${streak} consecutive ` +
|
|
3780
4008
|
`inferences — it cannot complete a turn. Last reason: ${reason}`);
|
|
3781
|
-
//
|
|
3782
|
-
|
|
3783
|
-
|
|
3784
|
-
|
|
3785
|
-
method: 'POST',
|
|
3786
|
-
headers: { 'content-type': 'application/json' },
|
|
3787
|
-
body: JSON.stringify({
|
|
3788
|
-
content: `\u{1F6A8} **${agentName}** hard-down: ${streak} consecutive inference failures.\nLast reason: ${reason.slice(0, 500)}`,
|
|
3789
|
-
}),
|
|
3790
|
-
}).catch((err) => console.error('[inference-hard-down] webhook post failed:', err instanceof Error ? err.message : err));
|
|
3791
|
-
}
|
|
4009
|
+
// Ops alert (webhook + ops:alert trace), throttled per (agent, kind) —
|
|
4010
|
+
// a hard-down agent re-posts every ~15 min, not on every failed retry.
|
|
4011
|
+
// failures.log already got the per-exhaustion record above.
|
|
4012
|
+
this.opsAlert('hard-down', agentName, `${streak} consecutive inference failures. Last reason: ${reason}`, { skipLog: true });
|
|
3792
4013
|
// (4) Poison-history breaker: ONLY for `invalid_request` — a 400-class
|
|
3793
4014
|
// rejection of the history itself. Deliberately NOT keyed on
|
|
3794
4015
|
// `retryable === false`, which membrane also returns for auth (expired
|
|
@@ -4233,6 +4454,13 @@ export class AgentFramework {
|
|
|
4233
4454
|
attempt: params.attempt,
|
|
4234
4455
|
willRetry: connection.willReconnect,
|
|
4235
4456
|
});
|
|
4457
|
+
// The reconnect loop never gives up (backoff caps at ~300s), so
|
|
4458
|
+
// "the server is effectively down" is an attempt-count judgment:
|
|
4459
|
+
// 5 failed attempts ≈ a few minutes of outage. Throttled per
|
|
4460
|
+
// (serverId, kind), so a long outage re-posts every ~15 min.
|
|
4461
|
+
if (params.attempt >= 5) {
|
|
4462
|
+
this.opsAlert('mcpl-down', connection.id, `MCPL server unreachable (attempt ${params.attempt}): ${params.error}`);
|
|
4463
|
+
}
|
|
4236
4464
|
});
|
|
4237
4465
|
// A late response to a tools/call that already timed out, carrying
|
|
4238
4466
|
// stateful data the server advanced to. The host can't re-inject it (the
|