@steadwing/openalerts 0.2.4 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/dist/collections/collection-manager.d.ts +50 -0
- package/dist/collections/collection-manager.js +583 -0
- package/dist/collections/event-parser.d.ts +27 -0
- package/dist/collections/event-parser.js +321 -0
- package/dist/collections/index.d.ts +6 -0
- package/dist/collections/index.js +6 -0
- package/dist/collections/persistence.d.ts +25 -0
- package/dist/collections/persistence.js +213 -0
- package/dist/collections/types.d.ts +177 -0
- package/dist/collections/types.js +15 -0
- package/dist/core/engine.js +2 -1
- package/dist/core/index.d.ts +1 -0
- package/dist/core/rules.js +97 -0
- package/dist/core/types.d.ts +1 -1
- package/dist/index.js +410 -23
- package/dist/plugin/dashboard-html.js +33 -3
- package/dist/plugin/dashboard-routes.d.ts +7 -2
- package/dist/plugin/dashboard-routes.js +111 -3
- package/dist/plugin/gateway-client.js +15 -8
- package/openclaw.plugin.json +30 -0
- package/package.json +1 -1
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
export interface MonitorSession {
|
|
2
|
+
key: string;
|
|
3
|
+
agentId: string;
|
|
4
|
+
platform: string;
|
|
5
|
+
recipient: string;
|
|
6
|
+
isGroup: boolean;
|
|
7
|
+
lastActivityAt: number;
|
|
8
|
+
status: "idle" | "active" | "thinking";
|
|
9
|
+
spawnedBy?: string;
|
|
10
|
+
messageCount?: number;
|
|
11
|
+
totalCostUsd?: number;
|
|
12
|
+
totalInputTokens?: number;
|
|
13
|
+
totalOutputTokens?: number;
|
|
14
|
+
}
|
|
15
|
+
export type MonitorActionType = "start" | "streaming" | "complete" | "aborted" | "error" | "tool_call" | "tool_result";
|
|
16
|
+
export type MonitorActionEventType = "chat" | "agent" | "system";
|
|
17
|
+
export interface MonitorAction {
|
|
18
|
+
id: string;
|
|
19
|
+
runId: string;
|
|
20
|
+
sessionKey: string;
|
|
21
|
+
seq: number;
|
|
22
|
+
type: MonitorActionType;
|
|
23
|
+
eventType: MonitorActionEventType;
|
|
24
|
+
timestamp: number;
|
|
25
|
+
content?: string;
|
|
26
|
+
toolName?: string;
|
|
27
|
+
toolArgs?: unknown;
|
|
28
|
+
startedAt?: number;
|
|
29
|
+
endedAt?: number;
|
|
30
|
+
duration?: number;
|
|
31
|
+
inputTokens?: number;
|
|
32
|
+
outputTokens?: number;
|
|
33
|
+
stopReason?: string;
|
|
34
|
+
costUsd?: number;
|
|
35
|
+
model?: string;
|
|
36
|
+
provider?: string;
|
|
37
|
+
}
|
|
38
|
+
export type MonitorExecEventType = "started" | "output" | "completed";
|
|
39
|
+
export type MonitorExecProcessStatus = "running" | "completed" | "failed";
|
|
40
|
+
export interface MonitorExecOutputChunk {
|
|
41
|
+
id: string;
|
|
42
|
+
stream: "stdout" | "stderr" | string;
|
|
43
|
+
text: string;
|
|
44
|
+
timestamp: number;
|
|
45
|
+
}
|
|
46
|
+
export interface MonitorExecEvent {
|
|
47
|
+
id: string;
|
|
48
|
+
execId: string;
|
|
49
|
+
runId: string;
|
|
50
|
+
pid: number;
|
|
51
|
+
sessionId?: string;
|
|
52
|
+
sessionKey?: string;
|
|
53
|
+
eventType: MonitorExecEventType;
|
|
54
|
+
command?: string;
|
|
55
|
+
stream?: "stdout" | "stderr" | string;
|
|
56
|
+
output?: string;
|
|
57
|
+
startedAt?: number;
|
|
58
|
+
durationMs?: number;
|
|
59
|
+
exitCode?: number;
|
|
60
|
+
status?: string;
|
|
61
|
+
timestamp: number;
|
|
62
|
+
}
|
|
63
|
+
export interface MonitorExecProcess {
|
|
64
|
+
id: string;
|
|
65
|
+
runId: string;
|
|
66
|
+
pid: number;
|
|
67
|
+
command: string;
|
|
68
|
+
sessionId?: string;
|
|
69
|
+
sessionKey?: string;
|
|
70
|
+
status: MonitorExecProcessStatus;
|
|
71
|
+
startedAt: number;
|
|
72
|
+
completedAt?: number;
|
|
73
|
+
durationMs?: number;
|
|
74
|
+
exitCode?: number;
|
|
75
|
+
outputs: MonitorExecOutputChunk[];
|
|
76
|
+
outputTruncated?: boolean;
|
|
77
|
+
timestamp: number;
|
|
78
|
+
lastActivityAt: number;
|
|
79
|
+
}
|
|
80
|
+
export interface ChatEvent {
|
|
81
|
+
runId: string;
|
|
82
|
+
sessionKey: string;
|
|
83
|
+
seq: number;
|
|
84
|
+
state: "delta" | "final" | "aborted" | "error";
|
|
85
|
+
message?: unknown;
|
|
86
|
+
errorMessage?: string;
|
|
87
|
+
usage?: {
|
|
88
|
+
inputTokens?: number;
|
|
89
|
+
outputTokens?: number;
|
|
90
|
+
};
|
|
91
|
+
stopReason?: string;
|
|
92
|
+
}
|
|
93
|
+
export interface AgentEvent {
|
|
94
|
+
runId: string;
|
|
95
|
+
seq: number;
|
|
96
|
+
stream: string;
|
|
97
|
+
ts: number;
|
|
98
|
+
data: Record<string, unknown>;
|
|
99
|
+
sessionKey?: string;
|
|
100
|
+
}
|
|
101
|
+
export interface ExecStartedEvent {
|
|
102
|
+
pid: number;
|
|
103
|
+
command: string;
|
|
104
|
+
sessionId: string;
|
|
105
|
+
runId: string;
|
|
106
|
+
startedAt: number;
|
|
107
|
+
}
|
|
108
|
+
export interface ExecOutputEvent {
|
|
109
|
+
pid: number;
|
|
110
|
+
runId: string;
|
|
111
|
+
sessionId?: string;
|
|
112
|
+
stream: "stdout" | "stderr" | string;
|
|
113
|
+
output: string;
|
|
114
|
+
}
|
|
115
|
+
export interface ExecCompletedEvent {
|
|
116
|
+
pid: number;
|
|
117
|
+
runId: string;
|
|
118
|
+
sessionId?: string;
|
|
119
|
+
exitCode: number;
|
|
120
|
+
durationMs: number;
|
|
121
|
+
status: string;
|
|
122
|
+
}
|
|
123
|
+
export declare function parseSessionKey(key: string): {
|
|
124
|
+
agentId: string;
|
|
125
|
+
platform: string;
|
|
126
|
+
recipient: string;
|
|
127
|
+
isGroup: boolean;
|
|
128
|
+
};
|
|
129
|
+
export interface CollectionStats {
|
|
130
|
+
sessions: number;
|
|
131
|
+
actions: number;
|
|
132
|
+
execs: number;
|
|
133
|
+
runSessionMapSize: number;
|
|
134
|
+
totalCostUsd?: number;
|
|
135
|
+
}
|
|
136
|
+
export interface DiagnosticUsageEvent {
|
|
137
|
+
type: "model.usage";
|
|
138
|
+
ts: number;
|
|
139
|
+
seq: number;
|
|
140
|
+
sessionKey?: string;
|
|
141
|
+
sessionId?: string;
|
|
142
|
+
channel?: string;
|
|
143
|
+
provider?: string;
|
|
144
|
+
model?: string;
|
|
145
|
+
usage: {
|
|
146
|
+
input?: number;
|
|
147
|
+
output?: number;
|
|
148
|
+
cacheRead?: number;
|
|
149
|
+
cacheWrite?: number;
|
|
150
|
+
promptTokens?: number;
|
|
151
|
+
total?: number;
|
|
152
|
+
};
|
|
153
|
+
context?: {
|
|
154
|
+
limit?: number;
|
|
155
|
+
used?: number;
|
|
156
|
+
};
|
|
157
|
+
costUsd?: number;
|
|
158
|
+
durationMs?: number;
|
|
159
|
+
}
|
|
160
|
+
export interface CostUsageTotals {
|
|
161
|
+
input: number;
|
|
162
|
+
output: number;
|
|
163
|
+
cacheRead: number;
|
|
164
|
+
cacheWrite: number;
|
|
165
|
+
totalTokens: number;
|
|
166
|
+
totalCost: number;
|
|
167
|
+
inputCost: number;
|
|
168
|
+
outputCost: number;
|
|
169
|
+
cacheReadCost: number;
|
|
170
|
+
cacheWriteCost: number;
|
|
171
|
+
missingCostEntries: number;
|
|
172
|
+
}
|
|
173
|
+
export interface CostUsageSummary {
|
|
174
|
+
totals: CostUsageTotals;
|
|
175
|
+
byModel?: Record<string, CostUsageTotals>;
|
|
176
|
+
bySession?: Record<string, CostUsageTotals>;
|
|
177
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
// ─── Session Types ───────────────────────────────────────────────────────────
|
|
2
|
+
// ─── Utility Functions ───────────────────────────────────────────────────────
|
|
3
|
+
export function parseSessionKey(key) {
|
|
4
|
+
// Format: "agent:main:discord:channel:1234567890"
|
|
5
|
+
// Or: "agent:main:telegram:group:12345"
|
|
6
|
+
// Or: "agent:main:whatsapp:+1234567890"
|
|
7
|
+
const parts = key.split(":");
|
|
8
|
+
const agentId = parts[1] || "unknown";
|
|
9
|
+
const platform = parts[2] || "unknown";
|
|
10
|
+
// Check if 4th part indicates a type (channel, group, dm, etc)
|
|
11
|
+
const hasType = ["channel", "group", "dm", "thread"].includes(parts[3] || "");
|
|
12
|
+
const isGroup = parts[3] === "group" || parts[3] === "channel";
|
|
13
|
+
const recipient = hasType ? parts.slice(3).join(":") : parts.slice(3).join(":");
|
|
14
|
+
return { agentId, platform, recipient, isGroup };
|
|
15
|
+
}
|
package/dist/core/engine.js
CHANGED
|
@@ -2,6 +2,7 @@ import { AlertDispatcher } from "./alert-channel.js";
|
|
|
2
2
|
import { OpenAlertsEventBus } from "./event-bus.js";
|
|
3
3
|
import { createEvaluatorState, processEvent, processWatchdogTick, warmFromHistory } from "./evaluator.js";
|
|
4
4
|
import { createPlatformSync } from "./platform.js";
|
|
5
|
+
import { ALL_RULES } from "./rules.js";
|
|
5
6
|
import { appendEvent, pruneLog, readAllEvents, readRecentEvents } from "./store.js";
|
|
6
7
|
import { DEFAULTS, } from "./types.js";
|
|
7
8
|
/**
|
|
@@ -88,7 +89,7 @@ export class OpenAlertsEngine {
|
|
|
88
89
|
const channelNames = this.dispatcher.hasChannels
|
|
89
90
|
? `${this.dispatcher.channelCount} channel(s)`
|
|
90
91
|
: "log-only (no alert channels)";
|
|
91
|
-
this.logger.info(`${this.logPrefix}: started, ${channelNames},
|
|
92
|
+
this.logger.info(`${this.logPrefix}: started, ${channelNames}, ${ALL_RULES.length} rules active`);
|
|
92
93
|
}
|
|
93
94
|
/** Ingest a universal event. Can be called directly or via the event bus. */
|
|
94
95
|
ingest(event) {
|
package/dist/core/index.d.ts
CHANGED
|
@@ -10,3 +10,4 @@ export { createLlmEnricher, type LlmEnricherOptions } from "./llm-enrichment.js"
|
|
|
10
10
|
export { formatAlertMessage, formatAlertsOutput, formatHealthOutput, } from "./formatter.js";
|
|
11
11
|
export { createPlatformSync, type PlatformSync } from "./platform.js";
|
|
12
12
|
export { BoundedMap, type BoundedMapOptions, type BoundedMapStats, } from "./bounded-map.js";
|
|
13
|
+
export type { MonitorSession, MonitorActionType, MonitorActionEventType, MonitorAction, MonitorExecEventType, MonitorExecProcessStatus, MonitorExecOutputChunk, MonitorExecEvent, MonitorExecProcess, CollectionStats, DiagnosticUsageEvent, CostUsageTotals, CostUsageSummary, } from "../collections/types.js";
|
package/dist/core/rules.js
CHANGED
|
@@ -22,6 +22,39 @@ function countInWindow(ctx, name, windowMs) {
|
|
|
22
22
|
const cutoff = ctx.now - windowMs;
|
|
23
23
|
return window.filter((e) => e.ts >= cutoff).length;
|
|
24
24
|
}
|
|
25
|
+
function sumInWindow(ctx, name, windowMs) {
|
|
26
|
+
const window = ctx.state.windows.get(name);
|
|
27
|
+
if (!window)
|
|
28
|
+
return 0;
|
|
29
|
+
const cutoff = ctx.now - windowMs;
|
|
30
|
+
let total = 0;
|
|
31
|
+
for (const entry of window) {
|
|
32
|
+
if (entry.ts >= cutoff) {
|
|
33
|
+
total += entry.value ?? 0;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return total;
|
|
37
|
+
}
|
|
38
|
+
function pushSummedBucket(ctx, name, value, bucketMs, maxEntries) {
|
|
39
|
+
if (!Number.isFinite(value) || value <= 0)
|
|
40
|
+
return;
|
|
41
|
+
const bucketTs = Math.floor(ctx.now / bucketMs) * bucketMs;
|
|
42
|
+
let window = ctx.state.windows.get(name);
|
|
43
|
+
if (!window) {
|
|
44
|
+
window = [];
|
|
45
|
+
ctx.state.windows.set(name, window);
|
|
46
|
+
}
|
|
47
|
+
const last = window[window.length - 1];
|
|
48
|
+
if (last && last.ts === bucketTs) {
|
|
49
|
+
last.value = (last.value ?? 0) + value;
|
|
50
|
+
}
|
|
51
|
+
else {
|
|
52
|
+
window.push({ ts: bucketTs, value });
|
|
53
|
+
}
|
|
54
|
+
if (window.length > maxEntries) {
|
|
55
|
+
window.splice(0, window.length - maxEntries);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
25
58
|
function getRuleThreshold(ctx, ruleId, defaultVal) {
|
|
26
59
|
return ctx.config.rules?.[ruleId]?.threshold ?? defaultVal;
|
|
27
60
|
}
|
|
@@ -234,6 +267,68 @@ const highErrorRate = {
|
|
|
234
267
|
};
|
|
235
268
|
},
|
|
236
269
|
};
|
|
270
|
+
// ─── Rule: cost-hourly-spike ────────────────────────────────────────────────
|
|
271
|
+
const costHourlySpike = {
|
|
272
|
+
id: "cost-hourly-spike",
|
|
273
|
+
defaultCooldownMs: 30 * 60 * 1000,
|
|
274
|
+
defaultThreshold: 5.0, // USD
|
|
275
|
+
evaluate(event, ctx) {
|
|
276
|
+
if (event.type !== "llm.token_usage")
|
|
277
|
+
return null;
|
|
278
|
+
if (!isRuleEnabled(ctx, "cost-hourly-spike"))
|
|
279
|
+
return null;
|
|
280
|
+
if (typeof event.costUsd !== "number" || !Number.isFinite(event.costUsd))
|
|
281
|
+
return null;
|
|
282
|
+
// Aggregate per-minute to keep 60m sums accurate under high event throughput.
|
|
283
|
+
pushSummedBucket(ctx, "cost-hourly-spike-usd", event.costUsd, 60_000, 120);
|
|
284
|
+
const hourlyUsd = sumInWindow(ctx, "cost-hourly-spike-usd", 60 * 60 * 1000);
|
|
285
|
+
const threshold = getRuleThreshold(ctx, "cost-hourly-spike", 5.0);
|
|
286
|
+
if (hourlyUsd <= threshold)
|
|
287
|
+
return null; // must exceed threshold
|
|
288
|
+
const fingerprint = "cost-hourly-spike";
|
|
289
|
+
return {
|
|
290
|
+
type: "alert",
|
|
291
|
+
id: makeAlertId("cost-hourly-spike", fingerprint, ctx.now),
|
|
292
|
+
ruleId: "cost-hourly-spike",
|
|
293
|
+
severity: "warn",
|
|
294
|
+
title: "Hourly LLM spend spike",
|
|
295
|
+
detail: `LLM spend reached $${hourlyUsd.toFixed(2)} in the last 60 minutes (threshold: $${threshold.toFixed(2)}).`,
|
|
296
|
+
ts: ctx.now,
|
|
297
|
+
fingerprint,
|
|
298
|
+
};
|
|
299
|
+
},
|
|
300
|
+
};
|
|
301
|
+
// ─── Rule: cost-daily-budget ────────────────────────────────────────────────
|
|
302
|
+
const costDailyBudget = {
|
|
303
|
+
id: "cost-daily-budget",
|
|
304
|
+
defaultCooldownMs: 6 * 60 * 60 * 1000,
|
|
305
|
+
defaultThreshold: 20.0, // USD
|
|
306
|
+
evaluate(event, ctx) {
|
|
307
|
+
if (event.type !== "llm.token_usage")
|
|
308
|
+
return null;
|
|
309
|
+
if (!isRuleEnabled(ctx, "cost-daily-budget"))
|
|
310
|
+
return null;
|
|
311
|
+
if (typeof event.costUsd !== "number" || !Number.isFinite(event.costUsd))
|
|
312
|
+
return null;
|
|
313
|
+
// Aggregate per-minute and retain >24h buckets for boundary-safe sums.
|
|
314
|
+
pushSummedBucket(ctx, "cost-daily-budget-usd", event.costUsd, 60_000, 1_500);
|
|
315
|
+
const dailyUsd = sumInWindow(ctx, "cost-daily-budget-usd", 24 * 60 * 60 * 1000);
|
|
316
|
+
const threshold = getRuleThreshold(ctx, "cost-daily-budget", 20.0);
|
|
317
|
+
if (dailyUsd <= threshold)
|
|
318
|
+
return null; // must exceed threshold
|
|
319
|
+
const fingerprint = "cost-daily-budget";
|
|
320
|
+
return {
|
|
321
|
+
type: "alert",
|
|
322
|
+
id: makeAlertId("cost-daily-budget", fingerprint, ctx.now),
|
|
323
|
+
ruleId: "cost-daily-budget",
|
|
324
|
+
severity: "error",
|
|
325
|
+
title: "Daily LLM budget exceeded",
|
|
326
|
+
detail: `LLM spend reached $${dailyUsd.toFixed(2)} in the last 24 hours (threshold: $${threshold.toFixed(2)}).`,
|
|
327
|
+
ts: ctx.now,
|
|
328
|
+
fingerprint,
|
|
329
|
+
};
|
|
330
|
+
},
|
|
331
|
+
};
|
|
237
332
|
// ─── Rule: tool-errors ───────────────────────────────────────────────────
|
|
238
333
|
const toolErrors = {
|
|
239
334
|
id: "tool-errors",
|
|
@@ -307,6 +402,8 @@ export const ALL_RULES = [
|
|
|
307
402
|
heartbeatFail,
|
|
308
403
|
queueDepth,
|
|
309
404
|
highErrorRate,
|
|
405
|
+
costHourlySpike,
|
|
406
|
+
costDailyBudget,
|
|
310
407
|
toolErrors,
|
|
311
408
|
gatewayDown,
|
|
312
409
|
];
|
package/dist/core/types.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export type AlertSeverity = "info" | "warn" | "error" | "critical";
|
|
2
|
-
export type OpenAlertsEventType = "llm.call" | "llm.error" | "llm.token_usage" | "tool.call" | "tool.error" | "agent.start" | "agent.end" | "agent.error" | "agent.stuck" | "session.start" | "session.end" | "session.stuck" | "infra.error" | "infra.heartbeat" | "infra.queue_depth" | "custom" | "watchdog.tick";
|
|
2
|
+
export type OpenAlertsEventType = "llm.call" | "llm.error" | "llm.token_usage" | "tool.call" | "tool.error" | "agent.start" | "agent.end" | "agent.error" | "agent.stuck" | "session.start" | "session.end" | "session.stuck" | "infra.error" | "infra.heartbeat" | "infra.queue_depth" | "exec.start" | "exec.output" | "exec.end" | "custom" | "watchdog.tick";
|
|
3
3
|
export type OpenAlertsEvent = {
|
|
4
4
|
type: OpenAlertsEventType;
|
|
5
5
|
ts: number;
|