@animalabs/agent-framework 0.6.2 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/agent.d.ts +3 -1
- package/dist/src/agent.d.ts.map +1 -1
- package/dist/src/agent.js +7 -2
- package/dist/src/agent.js.map +1 -1
- package/dist/src/framework.d.ts +60 -1
- package/dist/src/framework.d.ts.map +1 -1
- package/dist/src/framework.js +261 -20
- package/dist/src/framework.js.map +1 -1
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/modules/discord/index.d.ts +1 -0
- package/dist/src/modules/discord/index.d.ts.map +1 -1
- package/dist/src/modules/discord/index.js +4 -1
- package/dist/src/modules/discord/index.js.map +1 -1
- package/dist/src/modules/discord/types.d.ts +2 -0
- package/dist/src/modules/discord/types.d.ts.map +1 -1
- package/dist/src/modules/health/index.d.ts +2 -0
- package/dist/src/modules/health/index.d.ts.map +1 -1
- package/dist/src/modules/health/index.js +15 -4
- package/dist/src/modules/health/index.js.map +1 -1
- package/dist/src/timezone.d.ts +14 -0
- package/dist/src/timezone.d.ts.map +1 -0
- package/dist/src/timezone.js +74 -0
- package/dist/src/timezone.js.map +1 -0
- package/dist/src/types/agent.d.ts +5 -2
- package/dist/src/types/agent.d.ts.map +1 -1
- package/dist/src/types/framework.d.ts +46 -0
- package/dist/src/types/framework.d.ts.map +1 -1
- package/dist/src/types/trace.d.ts +9 -0
- package/dist/src/types/trace.d.ts.map +1 -1
- package/dist/test/agent-cache-ttl.test.js +18 -2
- package/dist/test/agent-cache-ttl.test.js.map +1 -1
- package/dist/test/health-snapshot.test.d.ts +2 -0
- package/dist/test/health-snapshot.test.d.ts.map +1 -0
- package/dist/test/health-snapshot.test.js +74 -0
- package/dist/test/health-snapshot.test.js.map +1 -0
- package/dist/test/inference-failure-observability.test.js +5 -0
- package/dist/test/inference-failure-observability.test.js.map +1 -1
- package/dist/test/maintenance-timer.test.d.ts +2 -0
- package/dist/test/maintenance-timer.test.d.ts.map +1 -0
- package/dist/test/maintenance-timer.test.js +114 -0
- package/dist/test/maintenance-timer.test.js.map +1 -0
- package/dist/test/ops-alert.test.d.ts +2 -0
- package/dist/test/ops-alert.test.d.ts.map +1 -0
- package/dist/test/ops-alert.test.js +152 -0
- package/dist/test/ops-alert.test.js.map +1 -0
- package/dist/test/timezone.test.d.ts +2 -0
- package/dist/test/timezone.test.d.ts.map +1 -0
- package/dist/test/timezone.test.js +18 -0
- package/dist/test/timezone.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/agent.ts +8 -3
- package/src/framework.ts +307 -22
- package/src/index.ts +1 -0
- package/src/modules/discord/index.ts +4 -1
- package/src/modules/discord/types.ts +3 -0
- package/src/modules/health/index.ts +20 -4
- package/src/timezone.ts +90 -0
- package/src/types/agent.ts +6 -2
- package/src/types/framework.ts +51 -0
- package/src/types/trace.ts +16 -0
package/package.json
CHANGED
package/src/agent.ts
CHANGED
|
@@ -47,7 +47,9 @@ export class Agent {
|
|
|
47
47
|
/** Refusal auto-rewind policy (see AgentConfig.refusalHandling). */
|
|
48
48
|
readonly refusalHandling: AgentConfig['refusalHandling'];
|
|
49
49
|
/** Prompt-cache TTL forwarded to the provider (see AgentConfig.cacheTtl). */
|
|
50
|
-
readonly cacheTtl: AgentConfig['cacheTtl']
|
|
50
|
+
readonly cacheTtl: NonNullable<AgentConfig['cacheTtl']>;
|
|
51
|
+
/** Provider-specific request parameters forwarded unchanged by Membrane. */
|
|
52
|
+
readonly providerParams?: Record<string, unknown>;
|
|
51
53
|
|
|
52
54
|
private _state: AgentState = { status: 'idle' };
|
|
53
55
|
private _inferenceStartedAt = 0;
|
|
@@ -75,7 +77,8 @@ export class Agent {
|
|
|
75
77
|
this.temperature = config.temperature;
|
|
76
78
|
this.thinking = config.thinking;
|
|
77
79
|
this.refusalHandling = config.refusalHandling;
|
|
78
|
-
this.cacheTtl = config.cacheTtl;
|
|
80
|
+
this.cacheTtl = config.cacheTtl ?? '1h';
|
|
81
|
+
this.providerParams = config.providerParams;
|
|
79
82
|
this.maxStreamTokens = config.maxStreamTokens ?? 150_000;
|
|
80
83
|
this.contextBudgetTokens = config.contextBudgetTokens;
|
|
81
84
|
this.contextManager = contextManager;
|
|
@@ -217,6 +220,7 @@ export class Agent {
|
|
|
217
220
|
...(this.thinking !== undefined && { thinking: this.thinking }),
|
|
218
221
|
},
|
|
219
222
|
tools: tools.length > 0 ? tools : undefined,
|
|
223
|
+
...(this.providerParams && { providerParams: this.providerParams }),
|
|
220
224
|
assistantParticipant: this.name,
|
|
221
225
|
};
|
|
222
226
|
|
|
@@ -357,7 +361,8 @@ export class Agent {
|
|
|
357
361
|
},
|
|
358
362
|
tools: availableTools.length > 0 ? availableTools : undefined,
|
|
359
363
|
promptCaching: true,
|
|
360
|
-
|
|
364
|
+
cacheTtl: this.cacheTtl,
|
|
365
|
+
...(this.providerParams && { providerParams: this.providerParams }),
|
|
361
366
|
assistantParticipant: this.name,
|
|
362
367
|
};
|
|
363
368
|
}
|
package/src/framework.ts
CHANGED
|
@@ -40,6 +40,9 @@ import type {
|
|
|
40
40
|
AgentState,
|
|
41
41
|
Module,
|
|
42
42
|
SpeechContext,
|
|
43
|
+
ContextMaintenanceRun,
|
|
44
|
+
ContextMaintenanceAgentRun,
|
|
45
|
+
ContextMaintenanceSnapshot,
|
|
43
46
|
} from './types/index.js';
|
|
44
47
|
import { ProcessQueueImpl } from './queue.js';
|
|
45
48
|
import { Agent } from './agent.js';
|
|
@@ -76,6 +79,7 @@ import type {
|
|
|
76
79
|
ChannelsIncomingParams,
|
|
77
80
|
} from './mcpl/types.js';
|
|
78
81
|
import type { ContextInjection } from '@animalabs/context-manager';
|
|
82
|
+
import { formatZonedDateTime, resolveTimeZone } from './timezone.js';
|
|
79
83
|
|
|
80
84
|
const FRAMEWORK_STATE_ID = 'framework/state';
|
|
81
85
|
const CONVERSATION_ROUTER_STATE_ID = 'framework/conversation-router';
|
|
@@ -144,6 +148,10 @@ class DefaultErrorPolicy implements ErrorPolicy {
|
|
|
144
148
|
|
|
145
149
|
/** Default sync interval in milliseconds */
|
|
146
150
|
const DEFAULT_SYNC_INTERVAL_MS = 1000;
|
|
151
|
+
/** Poll pending context-strategy maintenance independently of user activity. */
|
|
152
|
+
const DEFAULT_MAINTENANCE_INTERVAL_MS = 5000;
|
|
153
|
+
/** Bound one pass so a large backlog yields to inference and other agents. */
|
|
154
|
+
const MAINTENANCE_TICKS_PER_PASS = 8;
|
|
147
155
|
|
|
148
156
|
/**
|
|
149
157
|
* Extract fields the EventGate cares about from a ProcessEvent. The set of
|
|
@@ -253,6 +261,12 @@ export class AgentFramework {
|
|
|
253
261
|
private traceListeners: TraceEventListener[] = [];
|
|
254
262
|
private syncIntervalMs: number;
|
|
255
263
|
private syncTimer: ReturnType<typeof setInterval> | null = null;
|
|
264
|
+
private maintenanceIntervalMs: number;
|
|
265
|
+
private maintenanceTimer: ReturnType<typeof setInterval> | null = null;
|
|
266
|
+
private maintenancePass: Promise<void> | null = null;
|
|
267
|
+
private maintenanceRunId = 0;
|
|
268
|
+
private currentMaintenanceRun: ContextMaintenanceRun | null = null;
|
|
269
|
+
private maintenanceHistory: ContextMaintenanceRun[] = [];
|
|
256
270
|
/** Last time we console-warned about stale (busy-requeued) inference requests, per agent. */
|
|
257
271
|
private staleWarnAt = new Map<string, number>();
|
|
258
272
|
/** Per-agent last inference activity (epoch ms), for /healthz + doctor tooling. */
|
|
@@ -267,6 +281,19 @@ export class AgentFramework {
|
|
|
267
281
|
/** N consecutive failed inferences ⇒ the agent is treated as hard-down and
|
|
268
282
|
* escalated loudly to stderr. */
|
|
269
283
|
private readonly inferenceFailureEscalationThreshold = 3;
|
|
284
|
+
/** Per-(agent,kind) timestamp of the last ops webhook post. Throttles
|
|
285
|
+
* opsAlert() so a persistent failure re-posts once per cooldown window
|
|
286
|
+
* instead of on every occurrence. */
|
|
287
|
+
private opsAlertLastSent: Map<string, number> = new Map();
|
|
288
|
+
/** Cooldown between webhook posts for the same (agent, kind). */
|
|
289
|
+
private readonly opsAlertCooldownMs = 15 * 60_000;
|
|
290
|
+
/** Per-agent refusal bookkeeping for observability — exposed via
|
|
291
|
+
* healthSnapshot() and mirrored to failures.log / ops alerts. */
|
|
292
|
+
private refusalStats: Map<string, { total: number; byCategory: Record<string, number>; lastAt: number; lastCategory: string }> = new Map();
|
|
293
|
+
/** Per-agent count of consecutive refusals (reset on any non-refusal
|
|
294
|
+
* completion). Distinct from refusalRewinds, which budgets the auto-rewind
|
|
295
|
+
* loop — this one drives ops alerting. */
|
|
296
|
+
private refusalStreak: Map<string, number> = new Map();
|
|
270
297
|
/** Per-agent count of consecutive refusal-driven rewinds in the current turn
|
|
271
298
|
* chain (reset when a turn completes without a refusal). Bounds the auto
|
|
272
299
|
* rewind loop — see refusalHandling + rewindTriggeringTurn. */
|
|
@@ -348,6 +375,8 @@ export class AgentFramework {
|
|
|
348
375
|
|
|
349
376
|
// Session-level token usage tracking (always-on)
|
|
350
377
|
private usageTracker: UsageTracker;
|
|
378
|
+
/** Presentation-only wall-clock zone; persistence remains UTC/epoch. */
|
|
379
|
+
private readonly timeZone: string;
|
|
351
380
|
|
|
352
381
|
private constructor(
|
|
353
382
|
store: JsStore,
|
|
@@ -356,8 +385,10 @@ export class AgentFramework {
|
|
|
356
385
|
inferencePolicy: InferencePolicy,
|
|
357
386
|
errorPolicy: ErrorPolicy,
|
|
358
387
|
syncIntervalMs: number,
|
|
388
|
+
maintenanceIntervalMs: number,
|
|
359
389
|
processLoggingPersist: boolean,
|
|
360
|
-
processLoggingBroadcast: boolean
|
|
390
|
+
processLoggingBroadcast: boolean,
|
|
391
|
+
timeZone: string,
|
|
361
392
|
) {
|
|
362
393
|
this.store = store;
|
|
363
394
|
this.ownsStore = ownsStore;
|
|
@@ -365,8 +396,10 @@ export class AgentFramework {
|
|
|
365
396
|
this.inferencePolicy = inferencePolicy;
|
|
366
397
|
this.errorPolicy = errorPolicy;
|
|
367
398
|
this.syncIntervalMs = syncIntervalMs;
|
|
399
|
+
this.maintenanceIntervalMs = maintenanceIntervalMs;
|
|
368
400
|
this.processLoggingPersist = processLoggingPersist;
|
|
369
401
|
this.processLoggingBroadcast = processLoggingBroadcast;
|
|
402
|
+
this.timeZone = timeZone;
|
|
370
403
|
this.queue = new ProcessQueueImpl();
|
|
371
404
|
this.usageTracker = new UsageTracker({
|
|
372
405
|
emitTrace: (e: UsageUpdatedEvent) => this.emitTrace({ ...e }),
|
|
@@ -455,8 +488,10 @@ export class AgentFramework {
|
|
|
455
488
|
config.inferencePolicy ?? new DefaultInferencePolicy(),
|
|
456
489
|
config.errorPolicy ?? new DefaultErrorPolicy(),
|
|
457
490
|
config.syncIntervalMs ?? DEFAULT_SYNC_INTERVAL_MS,
|
|
491
|
+
config.maintenanceIntervalMs ?? DEFAULT_MAINTENANCE_INTERVAL_MS,
|
|
458
492
|
processLoggingPersist,
|
|
459
|
-
processLoggingBroadcast
|
|
493
|
+
processLoggingBroadcast,
|
|
494
|
+
resolveTimeZone(config.timeZone),
|
|
460
495
|
);
|
|
461
496
|
|
|
462
497
|
// Restore persisted usage data (if any) from prior session
|
|
@@ -610,6 +645,16 @@ export class AgentFramework {
|
|
|
610
645
|
}
|
|
611
646
|
}, this.syncIntervalMs);
|
|
612
647
|
}
|
|
648
|
+
|
|
649
|
+
if (this.maintenanceIntervalMs > 0) {
|
|
650
|
+
this.maintenanceTimer = setInterval(() => {
|
|
651
|
+
this.startQueuedMaintenance();
|
|
652
|
+
}, this.maintenanceIntervalMs);
|
|
653
|
+
this.maintenanceTimer.unref?.();
|
|
654
|
+
// Do not make a restored queue wait a full interval before its first
|
|
655
|
+
// attempt. MCPL/module initialization has completed before start().
|
|
656
|
+
this.startQueuedMaintenance();
|
|
657
|
+
}
|
|
613
658
|
}
|
|
614
659
|
|
|
615
660
|
/**
|
|
@@ -638,6 +683,14 @@ export class AgentFramework {
|
|
|
638
683
|
this.syncTimer = null;
|
|
639
684
|
}
|
|
640
685
|
|
|
686
|
+
if (this.maintenanceTimer) {
|
|
687
|
+
clearInterval(this.maintenanceTimer);
|
|
688
|
+
this.maintenanceTimer = null;
|
|
689
|
+
}
|
|
690
|
+
if (this.maintenancePass) {
|
|
691
|
+
await this.maintenancePass;
|
|
692
|
+
}
|
|
693
|
+
|
|
641
694
|
if (this.loopPromise) {
|
|
642
695
|
await this.loopPromise;
|
|
643
696
|
}
|
|
@@ -668,6 +721,103 @@ export class AgentFramework {
|
|
|
668
721
|
}
|
|
669
722
|
}
|
|
670
723
|
|
|
724
|
+
/**
|
|
725
|
+
* Advance queued ContextManager work without requiring a new message.
|
|
726
|
+
*
|
|
727
|
+
* Tool definitions are refreshed first because compression replays stored
|
|
728
|
+
* tool cycles and providers reject those requests when the corresponding
|
|
729
|
+
* schemas are absent. Passes never overlap; each agent gets a bounded drain
|
|
730
|
+
* so maintenance cannot monopolize the framework event loop.
|
|
731
|
+
*/
|
|
732
|
+
private startQueuedMaintenance(): void {
|
|
733
|
+
if (this.maintenancePass || !this.running) return;
|
|
734
|
+
const pass = this.runQueuedMaintenance().catch((error) => {
|
|
735
|
+
console.error(
|
|
736
|
+
'[context-maintenance] pass failed:',
|
|
737
|
+
error instanceof Error ? error.message : error,
|
|
738
|
+
);
|
|
739
|
+
});
|
|
740
|
+
this.maintenancePass = pass;
|
|
741
|
+
void pass.finally(() => {
|
|
742
|
+
if (this.maintenancePass === pass) this.maintenancePass = null;
|
|
743
|
+
});
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
private async runQueuedMaintenance(): Promise<void> {
|
|
747
|
+
const allTools = this.getAllTools();
|
|
748
|
+
const queued = [...this.agents.values()].flatMap((agent) => {
|
|
749
|
+
const cm = agent.getContextManager();
|
|
750
|
+
const tools = allTools.filter((tool) => agent.canUseTool(tool.name));
|
|
751
|
+
cm.setToolDefinitions(tools);
|
|
752
|
+
if (cm.isReady()) return [];
|
|
753
|
+
const pending = cm.getPendingWork()?.description;
|
|
754
|
+
const progress = this.contextProgress(cm);
|
|
755
|
+
const record: ContextMaintenanceAgentRun = {
|
|
756
|
+
agentName: agent.name,
|
|
757
|
+
startedAt: Date.now(),
|
|
758
|
+
ticks: 0,
|
|
759
|
+
readyBefore: false,
|
|
760
|
+
...(pending ? { pendingBefore: pending } : {}),
|
|
761
|
+
...(progress ? { progressBefore: progress } : {}),
|
|
762
|
+
};
|
|
763
|
+
return [{
|
|
764
|
+
agent,
|
|
765
|
+
cm,
|
|
766
|
+
record,
|
|
767
|
+
}];
|
|
768
|
+
});
|
|
769
|
+
if (queued.length === 0) return;
|
|
770
|
+
|
|
771
|
+
const run: ContextMaintenanceRun = {
|
|
772
|
+
id: ++this.maintenanceRunId,
|
|
773
|
+
startedAt: Date.now(),
|
|
774
|
+
agents: queued.map(item => item.record),
|
|
775
|
+
};
|
|
776
|
+
this.currentMaintenanceRun = run;
|
|
777
|
+
|
|
778
|
+
try {
|
|
779
|
+
await Promise.all(queued.map(async ({ agent, cm, record }) => {
|
|
780
|
+
try {
|
|
781
|
+
for (
|
|
782
|
+
let i = 0;
|
|
783
|
+
i < MAINTENANCE_TICKS_PER_PASS && this.running && !cm.isReady();
|
|
784
|
+
i++
|
|
785
|
+
) {
|
|
786
|
+
await cm.tick();
|
|
787
|
+
record.ticks++;
|
|
788
|
+
}
|
|
789
|
+
} catch (error) {
|
|
790
|
+
record.error = error instanceof Error ? error.message : String(error);
|
|
791
|
+
console.error(`[context-maintenance] agent=${agent.name} failed:`, record.error);
|
|
792
|
+
} finally {
|
|
793
|
+
const pending = cm.getPendingWork()?.description;
|
|
794
|
+
record.finishedAt = Date.now();
|
|
795
|
+
record.readyAfter = cm.isReady();
|
|
796
|
+
if (pending) record.pendingAfter = pending;
|
|
797
|
+
const progress = this.contextProgress(cm);
|
|
798
|
+
if (progress) record.progressAfter = progress;
|
|
799
|
+
}
|
|
800
|
+
}));
|
|
801
|
+
} finally {
|
|
802
|
+
run.finishedAt = Date.now();
|
|
803
|
+
run.agents.sort((a, b) => a.agentName.localeCompare(b.agentName));
|
|
804
|
+
this.currentMaintenanceRun = null;
|
|
805
|
+
this.maintenanceHistory.unshift(run);
|
|
806
|
+
if (this.maintenanceHistory.length > 50) this.maintenanceHistory.length = 50;
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
private contextProgress(cm: ContextManager): Record<string, unknown> | undefined {
|
|
811
|
+
const strategy = cm.getStrategy() as {
|
|
812
|
+
getProgressSnapshot?: () => unknown;
|
|
813
|
+
};
|
|
814
|
+
if (typeof strategy.getProgressSnapshot !== 'function') return undefined;
|
|
815
|
+
const progress = strategy.getProgressSnapshot();
|
|
816
|
+
return progress && typeof progress === 'object'
|
|
817
|
+
? progress as Record<string, unknown>
|
|
818
|
+
: undefined;
|
|
819
|
+
}
|
|
820
|
+
|
|
671
821
|
/**
|
|
672
822
|
* Push a process event to the queue.
|
|
673
823
|
*/
|
|
@@ -790,6 +940,28 @@ export class AgentFramework {
|
|
|
790
940
|
return [...moduleTools, ...this.mcplTools, ...channelTools, ...gateTools];
|
|
791
941
|
}
|
|
792
942
|
|
|
943
|
+
/** Counts-only context-maintenance diagnostics for authenticated debug UIs. */
|
|
944
|
+
getContextMaintenanceSnapshot(): ContextMaintenanceSnapshot {
|
|
945
|
+
const agents = [...this.agents.values()].map((agent) => {
|
|
946
|
+
const cm = agent.getContextManager();
|
|
947
|
+
const pending = cm.getPendingWork()?.description;
|
|
948
|
+
const progress = this.contextProgress(cm);
|
|
949
|
+
return {
|
|
950
|
+
agentName: agent.name,
|
|
951
|
+
ready: cm.isReady(),
|
|
952
|
+
...(pending ? { pending } : {}),
|
|
953
|
+
...(progress ? { progress } : {}),
|
|
954
|
+
};
|
|
955
|
+
});
|
|
956
|
+
return structuredClone({
|
|
957
|
+
intervalMs: this.maintenanceIntervalMs,
|
|
958
|
+
ticksPerPass: MAINTENANCE_TICKS_PER_PASS,
|
|
959
|
+
current: this.currentMaintenanceRun,
|
|
960
|
+
history: this.maintenanceHistory,
|
|
961
|
+
agents,
|
|
962
|
+
});
|
|
963
|
+
}
|
|
964
|
+
|
|
793
965
|
/**
|
|
794
966
|
* Build the membrane-normalized request that WOULD be emitted if `agentName`
|
|
795
967
|
* were activated right now — WITHOUT running inference, opening a stream, or
|
|
@@ -3544,6 +3716,8 @@ export class AgentFramework {
|
|
|
3544
3716
|
(stopDetails?.explanation ? ` explanation=${stopDetails.explanation}` : ''),
|
|
3545
3717
|
);
|
|
3546
3718
|
|
|
3719
|
+
this.noteRefusal(agent.name, category, tokenUsage);
|
|
3720
|
+
|
|
3547
3721
|
// Rewind: excise the turn that fed the refusal, drop a
|
|
3548
3722
|
// metadata-only marker, and retry — keeping the agent on its own
|
|
3549
3723
|
// model instead of substituting a fallback. Driven either by an
|
|
@@ -3621,6 +3795,7 @@ export class AgentFramework {
|
|
|
3621
3795
|
this.refusalRewinds.set(agent.name, 0);
|
|
3622
3796
|
}
|
|
3623
3797
|
this.rewindEpisode.delete(agent.name);
|
|
3798
|
+
this.refusalStreak.delete(agent.name);
|
|
3624
3799
|
}
|
|
3625
3800
|
|
|
3626
3801
|
// Dispatch speech (and thoughts if any)
|
|
@@ -4390,10 +4565,108 @@ export class AgentFramework {
|
|
|
4390
4565
|
status: agent.state.status,
|
|
4391
4566
|
consecutiveInferenceFailures: this.consecutiveInferenceFailures.get(name) ?? 0,
|
|
4392
4567
|
lastInference: this.lastInferenceAt.get(name) ?? null,
|
|
4568
|
+
refusalStats: this.refusalStats.get(name) ?? null,
|
|
4393
4569
|
})),
|
|
4394
4570
|
};
|
|
4395
4571
|
}
|
|
4396
4572
|
|
|
4573
|
+
/** Append a structured JSONL record to logs/failures.log (best-effort).
|
|
4574
|
+
* Durable and independent of journald/unit log redirects — this is what
|
|
4575
|
+
* connectome-doctor and fleet tooling read. Always stamps `at`. Legacy
|
|
4576
|
+
* records carry {at, agent, consecutive, reason}; new records add `kind`
|
|
4577
|
+
* and kind-specific fields (additive only — doctor parses by regex). */
|
|
4578
|
+
private logFailure(record: Record<string, unknown>): void {
|
|
4579
|
+
try {
|
|
4580
|
+
mkdirSync('logs', { recursive: true });
|
|
4581
|
+
appendFileSync(
|
|
4582
|
+
'logs/failures.log',
|
|
4583
|
+
JSON.stringify({ at: new Date().toISOString(), ...record }) + '\n',
|
|
4584
|
+
);
|
|
4585
|
+
} catch { /* best-effort */ }
|
|
4586
|
+
}
|
|
4587
|
+
|
|
4588
|
+
/**
|
|
4589
|
+
* Ops alert — the one escalation path for "a human should hear about this":
|
|
4590
|
+
* (1) durable failures.log record (unless the caller already wrote one),
|
|
4591
|
+
* (2) an `ops:alert` trace so authorized observers get it on the wire,
|
|
4592
|
+
* (3) if CONNECTOME_OPS_WEBHOOK is set, a Discord post throttled to one per
|
|
4593
|
+
* (agent, kind) per cooldown window — a persistent failure re-posts
|
|
4594
|
+
* every ~15 min instead of flooding the channel on every occurrence.
|
|
4595
|
+
* See connectome docs/observability.md.
|
|
4596
|
+
*/
|
|
4597
|
+
private opsAlert(
|
|
4598
|
+
kind: string,
|
|
4599
|
+
agentName: string,
|
|
4600
|
+
message: string,
|
|
4601
|
+
opts?: { data?: Record<string, unknown>; skipLog?: boolean },
|
|
4602
|
+
): void {
|
|
4603
|
+
if (!opts?.skipLog) {
|
|
4604
|
+
this.logFailure({ agent: agentName, kind, reason: message, ...opts?.data });
|
|
4605
|
+
}
|
|
4606
|
+
this.emitTrace({ type: 'ops:alert', kind, agentName, message, data: opts?.data });
|
|
4607
|
+
|
|
4608
|
+
const hook = process.env.CONNECTOME_OPS_WEBHOOK;
|
|
4609
|
+
if (!hook) return;
|
|
4610
|
+
const key = `${agentName}:${kind}`;
|
|
4611
|
+
const now = Date.now();
|
|
4612
|
+
if (now - (this.opsAlertLastSent.get(key) ?? 0) < this.opsAlertCooldownMs) return;
|
|
4613
|
+
// Stamp BEFORE the async post so a burst can't race past the cooldown.
|
|
4614
|
+
this.opsAlertLastSent.set(key, now);
|
|
4615
|
+
fetch(hook, {
|
|
4616
|
+
method: 'POST',
|
|
4617
|
+
headers: { 'content-type': 'application/json' },
|
|
4618
|
+
body: JSON.stringify({
|
|
4619
|
+
content: `\u{1F6A8} **${agentName}** ${kind}: ${message.slice(0, 500)}`,
|
|
4620
|
+
}),
|
|
4621
|
+
}).catch((err) => console.error(
|
|
4622
|
+
`[ops-alert] webhook post failed (${key}):`,
|
|
4623
|
+
err instanceof Error ? err.message : err,
|
|
4624
|
+
));
|
|
4625
|
+
}
|
|
4626
|
+
|
|
4627
|
+
/**
|
|
4628
|
+
* Record a model refusal for observability: per-agent stats + a
|
|
4629
|
+
* consecutive-refusal streak (exposed via healthSnapshot), a durable
|
|
4630
|
+
* failures.log record for EVERY refusal (previously invisible outside
|
|
4631
|
+
* stderr), and a throttled ops alert once refusals repeat — one refusal is
|
|
4632
|
+
* often survivable (auto-rewind may clear it); two in a row means stuck.
|
|
4633
|
+
* The streak is reset by any non-refusal completion in the stream driver.
|
|
4634
|
+
*/
|
|
4635
|
+
private noteRefusal(
|
|
4636
|
+
agentName: string,
|
|
4637
|
+
category: string,
|
|
4638
|
+
tokens?: { input: number; output: number },
|
|
4639
|
+
): number {
|
|
4640
|
+
const stats = this.refusalStats.get(agentName)
|
|
4641
|
+
?? { total: 0, byCategory: {}, lastAt: 0, lastCategory: '' };
|
|
4642
|
+
stats.total += 1;
|
|
4643
|
+
stats.byCategory[category] = (stats.byCategory[category] ?? 0) + 1;
|
|
4644
|
+
stats.lastAt = Date.now();
|
|
4645
|
+
stats.lastCategory = category;
|
|
4646
|
+
this.refusalStats.set(agentName, stats);
|
|
4647
|
+
|
|
4648
|
+
const streak = (this.refusalStreak.get(agentName) ?? 0) + 1;
|
|
4649
|
+
this.refusalStreak.set(agentName, streak);
|
|
4650
|
+
|
|
4651
|
+
this.logFailure({
|
|
4652
|
+
agent: agentName,
|
|
4653
|
+
kind: 'refusal',
|
|
4654
|
+
category,
|
|
4655
|
+
streak,
|
|
4656
|
+
locus: this.channelRegistry?.buildChannelContext()?.incoming?.channelId ?? null,
|
|
4657
|
+
tokens: tokens ?? null,
|
|
4658
|
+
});
|
|
4659
|
+
if (streak >= 2) {
|
|
4660
|
+
this.opsAlert(
|
|
4661
|
+
'refusal',
|
|
4662
|
+
agentName,
|
|
4663
|
+
`refusal streak ${streak}, category=${category}`,
|
|
4664
|
+
{ skipLog: true, data: { category, streak } },
|
|
4665
|
+
);
|
|
4666
|
+
}
|
|
4667
|
+
return streak;
|
|
4668
|
+
}
|
|
4669
|
+
|
|
4397
4670
|
private noteInferenceExhausted(
|
|
4398
4671
|
agentName: string,
|
|
4399
4672
|
reason: string,
|
|
@@ -4409,14 +4682,8 @@ export class AgentFramework {
|
|
|
4409
4682
|
|
|
4410
4683
|
// (1b) Machine-greppable durable record, independent of journald/unit log
|
|
4411
4684
|
// redirects: logs/failures.log under the host's working directory. This is
|
|
4412
|
-
// what connectome-doctor reads.
|
|
4413
|
-
|
|
4414
|
-
mkdirSync('logs', { recursive: true });
|
|
4415
|
-
appendFileSync(
|
|
4416
|
-
'logs/failures.log',
|
|
4417
|
-
JSON.stringify({ at: new Date().toISOString(), agent: agentName, consecutive: streak, reason }) + '\n',
|
|
4418
|
-
);
|
|
4419
|
-
} catch { /* best-effort */ }
|
|
4685
|
+
// what connectome-doctor reads. Legacy fields kept; `kind` is additive.
|
|
4686
|
+
this.logFailure({ agent: agentName, consecutive: streak, reason, kind: 'inference-exhausted' });
|
|
4420
4687
|
|
|
4421
4688
|
// (2) Agent-facing chronicle marker (no inference triggered → no loop).
|
|
4422
4689
|
const agent = this.agents.get(agentName);
|
|
@@ -4447,17 +4714,15 @@ export class AgentFramework {
|
|
|
4447
4714
|
`inferences — it cannot complete a turn. Last reason: ${reason}`,
|
|
4448
4715
|
);
|
|
4449
4716
|
|
|
4450
|
-
//
|
|
4451
|
-
|
|
4452
|
-
|
|
4453
|
-
|
|
4454
|
-
|
|
4455
|
-
|
|
4456
|
-
|
|
4457
|
-
|
|
4458
|
-
|
|
4459
|
-
}).catch((err) => console.error('[inference-hard-down] webhook post failed:', err instanceof Error ? err.message : err));
|
|
4460
|
-
}
|
|
4717
|
+
// Ops alert (webhook + ops:alert trace), throttled per (agent, kind) —
|
|
4718
|
+
// a hard-down agent re-posts every ~15 min, not on every failed retry.
|
|
4719
|
+
// failures.log already got the per-exhaustion record above.
|
|
4720
|
+
this.opsAlert(
|
|
4721
|
+
'hard-down',
|
|
4722
|
+
agentName,
|
|
4723
|
+
`${streak} consecutive inference failures. Last reason: ${reason}`,
|
|
4724
|
+
{ skipLog: true },
|
|
4725
|
+
);
|
|
4461
4726
|
|
|
4462
4727
|
// (4) Poison-history breaker: ONLY for `invalid_request` — a 400-class
|
|
4463
4728
|
// rejection of the history itself. Deliberately NOT keyed on
|
|
@@ -5030,6 +5295,17 @@ export class AgentFramework {
|
|
|
5030
5295
|
attempt: params.attempt,
|
|
5031
5296
|
willRetry: connection.willReconnect,
|
|
5032
5297
|
});
|
|
5298
|
+
// The reconnect loop never gives up (backoff caps at ~300s), so
|
|
5299
|
+
// "the server is effectively down" is an attempt-count judgment:
|
|
5300
|
+
// 5 failed attempts ≈ a few minutes of outage. Throttled per
|
|
5301
|
+
// (serverId, kind), so a long outage re-posts every ~15 min.
|
|
5302
|
+
if (params.attempt >= 5) {
|
|
5303
|
+
this.opsAlert(
|
|
5304
|
+
'mcpl-down',
|
|
5305
|
+
connection.id,
|
|
5306
|
+
`MCPL server unreachable (attempt ${params.attempt}): ${params.error}`,
|
|
5307
|
+
);
|
|
5308
|
+
}
|
|
5033
5309
|
});
|
|
5034
5310
|
|
|
5035
5311
|
// A late response to a tools/call that already timed out, carrying
|
|
@@ -5494,7 +5770,16 @@ export class AgentFramework {
|
|
|
5494
5770
|
|
|
5495
5771
|
console.error(`[sleep] agent=${agentName} seconds=${seconds} announce=${announce} until=${new Date(until).toISOString()}`);
|
|
5496
5772
|
// endTurn: the agent stops here and goes idle for the duration.
|
|
5497
|
-
finish({
|
|
5773
|
+
finish({
|
|
5774
|
+
success: true,
|
|
5775
|
+
data: {
|
|
5776
|
+
sleepingFor: human,
|
|
5777
|
+
until,
|
|
5778
|
+
untilLocal: formatZonedDateTime(until, this.timeZone),
|
|
5779
|
+
timeZone: this.timeZone,
|
|
5780
|
+
},
|
|
5781
|
+
endTurn: true,
|
|
5782
|
+
});
|
|
5498
5783
|
}
|
|
5499
5784
|
|
|
5500
5785
|
/** Aggregate the event-tag vocabulary: reserved chat:* core + each connected
|
package/src/index.ts
CHANGED
|
@@ -7,6 +7,7 @@ export { Agent } from './agent.js';
|
|
|
7
7
|
export type { StartStreamResult } from './agent.js';
|
|
8
8
|
export { ProcessQueueImpl } from './queue.js';
|
|
9
9
|
export { ModuleRegistry } from './module-registry.js';
|
|
10
|
+
export { formatZonedDateTime, formatZonedTime, isValidTimeZone, resolveTimeZone } from './timezone.js';
|
|
10
11
|
|
|
11
12
|
// Built-in modules
|
|
12
13
|
export * from './modules/index.js';
|
|
@@ -41,6 +41,7 @@ import type {
|
|
|
41
41
|
ListChannelsInput,
|
|
42
42
|
FetchHistoryInput,
|
|
43
43
|
} from './types.js';
|
|
44
|
+
import { formatZonedDateTime, resolveTimeZone } from '../../timezone.js';
|
|
44
45
|
|
|
45
46
|
// Re-export event types for consumers who want to handle them
|
|
46
47
|
export type { DiscordMessageEvent, DiscordEditEvent, DiscordDeleteEvent } from './types.js';
|
|
@@ -99,6 +100,7 @@ export class DiscordModule implements Module {
|
|
|
99
100
|
readonly name = 'discord';
|
|
100
101
|
|
|
101
102
|
private config: DiscordModuleConfig;
|
|
103
|
+
private readonly timeZone: string;
|
|
102
104
|
private ctx: ModuleContext | null = null;
|
|
103
105
|
private client: DiscordClientInterface;
|
|
104
106
|
private state: DiscordModuleState = {
|
|
@@ -126,6 +128,7 @@ export class DiscordModule implements Module {
|
|
|
126
128
|
constructor(client: DiscordClientInterface, config: DiscordModuleConfig) {
|
|
127
129
|
this.client = client;
|
|
128
130
|
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
131
|
+
this.timeZone = resolveTimeZone(config.timeZone);
|
|
129
132
|
}
|
|
130
133
|
|
|
131
134
|
async start(ctx: ModuleContext): Promise<void> {
|
|
@@ -1190,7 +1193,7 @@ export class DiscordModule implements Module {
|
|
|
1190
1193
|
authorId: m.authorId,
|
|
1191
1194
|
isBot: m.isBot,
|
|
1192
1195
|
content: m.content,
|
|
1193
|
-
timestamp: m.timestamp.
|
|
1196
|
+
timestamp: formatZonedDateTime(m.timestamp, this.timeZone),
|
|
1194
1197
|
replyTo: m.replyTo,
|
|
1195
1198
|
})),
|
|
1196
1199
|
},
|
|
@@ -9,6 +9,9 @@ import type { ContentBlock } from '@animalabs/membrane';
|
|
|
9
9
|
// ============================================================================
|
|
10
10
|
|
|
11
11
|
export interface DiscordModuleConfig {
|
|
12
|
+
/** IANA zone for timestamps returned to the agent by fetch_history. */
|
|
13
|
+
timeZone?: string;
|
|
14
|
+
|
|
12
15
|
/** Discord bot token */
|
|
13
16
|
token: string;
|
|
14
17
|
|
|
@@ -24,8 +24,12 @@ import type { Module, ModuleContext } from '../../types/module.js';
|
|
|
24
24
|
import type { ToolDefinition, ToolCall, ToolResult, ProcessEvent } from '../../types/events.js';
|
|
25
25
|
import type { EventResponse, ProcessState } from '../../types/module.js';
|
|
26
26
|
import type { AgentFramework } from '../../framework.js';
|
|
27
|
+
import { formatZonedDateTime, resolveTimeZone } from '../../timezone.js';
|
|
27
28
|
|
|
28
29
|
export interface HealthModuleConfig {
|
|
30
|
+
/** IANA zone for timestamps returned by the snapshot tool. */
|
|
31
|
+
timeZone?: string;
|
|
32
|
+
|
|
29
33
|
/**
|
|
30
34
|
* Default number of recent inferences to summarize when the tool is
|
|
31
35
|
* called without an explicit limit. Default: 20.
|
|
@@ -70,6 +74,7 @@ export class HealthModule implements Module {
|
|
|
70
74
|
|
|
71
75
|
constructor(config: HealthModuleConfig = {}) {
|
|
72
76
|
this.config = {
|
|
77
|
+
timeZone: resolveTimeZone(config.timeZone),
|
|
73
78
|
defaultLookback: config.defaultLookback ?? 20,
|
|
74
79
|
subagentStateId: config.subagentStateId ?? 'modules/subagent/state',
|
|
75
80
|
};
|
|
@@ -162,10 +167,20 @@ export class HealthModule implements Module {
|
|
|
162
167
|
const recent = this.framework.queryInferenceLogs({ limit: lookback });
|
|
163
168
|
const recentEntries = recent.entries ?? [];
|
|
164
169
|
|
|
170
|
+
// Presentation-boundary guard: queryInferenceLogs casts raw persisted
|
|
171
|
+
// JSON without validation, so a legacy/corrupt record can carry a missing
|
|
172
|
+
// or non-numeric timestamp. formatZonedDateTime throws on invalid dates
|
|
173
|
+
// (correctly — misuse should be loud), but one bad record must not sink
|
|
174
|
+
// the whole snapshot: degrade to the raw value (or null) instead.
|
|
175
|
+
const safeZoned = (ts: unknown): unknown =>
|
|
176
|
+
typeof ts === 'number' && Number.isFinite(ts)
|
|
177
|
+
? formatZonedDateTime(ts, this.config.timeZone)
|
|
178
|
+
: ts ?? null;
|
|
179
|
+
|
|
165
180
|
const projectSummary = (e: typeof recentEntries[number]) => {
|
|
166
181
|
const s = e.summary ?? e.entry;
|
|
167
182
|
return {
|
|
168
|
-
timestamp: s.timestamp,
|
|
183
|
+
timestamp: safeZoned(s.timestamp),
|
|
169
184
|
agentName: s.agentName,
|
|
170
185
|
success: s.success,
|
|
171
186
|
error: s.error,
|
|
@@ -224,8 +239,8 @@ export class HealthModule implements Module {
|
|
|
224
239
|
name: sa.name,
|
|
225
240
|
type: sa.type,
|
|
226
241
|
status: sa.status,
|
|
227
|
-
startedAt: startedAt > 0 ?
|
|
228
|
-
lastActivityAt: lastActivityAt > 0 ?
|
|
242
|
+
startedAt: startedAt > 0 ? formatZonedDateTime(startedAt, this.config.timeZone) : null,
|
|
243
|
+
lastActivityAt: lastActivityAt > 0 ? formatZonedDateTime(lastActivityAt, this.config.timeZone) : null,
|
|
229
244
|
runtimeSeconds: startedAt > 0 ? Math.floor((now - startedAt) / 1000) : null,
|
|
230
245
|
silentSeconds: lastActivityAt > 0 ? Math.floor((now - lastActivityAt) / 1000) : null,
|
|
231
246
|
toolCallsCount: sa.toolCallsCount,
|
|
@@ -244,7 +259,8 @@ export class HealthModule implements Module {
|
|
|
244
259
|
const modules = this.ctx ? this.listModulesViaContext() : [];
|
|
245
260
|
|
|
246
261
|
return {
|
|
247
|
-
timestamp:
|
|
262
|
+
timestamp: formatZonedDateTime(Date.now(), this.config.timeZone),
|
|
263
|
+
timeZone: this.config.timeZone,
|
|
248
264
|
window: {
|
|
249
265
|
lookback,
|
|
250
266
|
inferencesInWindow: recentEntries.length,
|