clawdoctor 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -0
- package/dist/alerters/telegram.d.ts +21 -0
- package/dist/alerters/telegram.d.ts.map +1 -0
- package/dist/alerters/telegram.js +87 -0
- package/dist/alerters/telegram.js.map +1 -0
- package/dist/config.d.ts +42 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +77 -0
- package/dist/config.js.map +1 -0
- package/dist/daemon.d.ts +21 -0
- package/dist/daemon.d.ts.map +1 -0
- package/dist/daemon.js +172 -0
- package/dist/daemon.js.map +1 -0
- package/dist/healers/base.d.ts +15 -0
- package/dist/healers/base.d.ts.map +1 -0
- package/dist/healers/base.js +25 -0
- package/dist/healers/base.js.map +1 -0
- package/dist/healers/cron.d.ts +6 -0
- package/dist/healers/cron.d.ts.map +1 -0
- package/dist/healers/cron.js +26 -0
- package/dist/healers/cron.js.map +1 -0
- package/dist/healers/process.d.ts +6 -0
- package/dist/healers/process.d.ts.map +1 -0
- package/dist/healers/process.js +67 -0
- package/dist/healers/process.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +283 -0
- package/dist/index.js.map +1 -0
- package/dist/store.d.ts +22 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +97 -0
- package/dist/store.js.map +1 -0
- package/dist/test/config.test.d.ts +2 -0
- package/dist/test/config.test.d.ts.map +1 -0
- package/dist/test/config.test.js +90 -0
- package/dist/test/config.test.js.map +1 -0
- package/dist/test/store.test.d.ts +2 -0
- package/dist/test/store.test.d.ts.map +1 -0
- package/dist/test/store.test.js +89 -0
- package/dist/test/store.test.js.map +1 -0
- package/dist/test/telegram.test.d.ts +2 -0
- package/dist/test/telegram.test.d.ts.map +1 -0
- package/dist/test/telegram.test.js +107 -0
- package/dist/test/telegram.test.js.map +1 -0
- package/dist/test/watchers.test.d.ts +2 -0
- package/dist/test/watchers.test.d.ts.map +1 -0
- package/dist/test/watchers.test.js +194 -0
- package/dist/test/watchers.test.js.map +1 -0
- package/dist/utils.d.ts +18 -0
- package/dist/utils.d.ts.map +1 -0
- package/dist/utils.js +62 -0
- package/dist/utils.js.map +1 -0
- package/dist/watchers/auth.d.ts +10 -0
- package/dist/watchers/auth.d.ts.map +1 -0
- package/dist/watchers/auth.js +79 -0
- package/dist/watchers/auth.js.map +1 -0
- package/dist/watchers/base.d.ts +22 -0
- package/dist/watchers/base.d.ts.map +1 -0
- package/dist/watchers/base.js +39 -0
- package/dist/watchers/base.js.map +1 -0
- package/dist/watchers/cost.d.ts +9 -0
- package/dist/watchers/cost.d.ts.map +1 -0
- package/dist/watchers/cost.js +151 -0
- package/dist/watchers/cost.js.map +1 -0
- package/dist/watchers/cron.d.ts +7 -0
- package/dist/watchers/cron.d.ts.map +1 -0
- package/dist/watchers/cron.js +79 -0
- package/dist/watchers/cron.js.map +1 -0
- package/dist/watchers/gateway.d.ts +7 -0
- package/dist/watchers/gateway.d.ts.map +1 -0
- package/dist/watchers/gateway.js +33 -0
- package/dist/watchers/gateway.js.map +1 -0
- package/dist/watchers/session.d.ts +7 -0
- package/dist/watchers/session.d.ts.map +1 -0
- package/dist/watchers/session.js +112 -0
- package/dist/watchers/session.js.map +1 -0
- package/package.json +46 -0
package/README.md
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
# clawdoctor
|
|
2
|
+
|
|
3
|
+
Self-healing monitor for OpenClaw. Watches your gateway, crons, and agent sessions, sends Telegram alerts, and auto-fixes what it can.
|
|
4
|
+
|
|
5
|
+
Built by people who run 20+ OpenClaw agents in production and got tired of checking if things were still alive.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm install -g clawdoctor
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Quick Start
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
# Configure
|
|
17
|
+
clawdoctor init
|
|
18
|
+
|
|
19
|
+
# Start monitoring
|
|
20
|
+
clawdoctor start
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
## Commands
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
clawdoctor init # Interactive setup
|
|
27
|
+
clawdoctor start # Start monitoring daemon
|
|
28
|
+
clawdoctor start --dry-run # Run without taking healing actions
|
|
29
|
+
clawdoctor stop # Stop daemon
|
|
30
|
+
clawdoctor status # Live health check of all monitors
|
|
31
|
+
clawdoctor log # Show recent events from local database
|
|
32
|
+
clawdoctor log -n 100 # Show 100 events
|
|
33
|
+
clawdoctor log -w GatewayWatcher -s critical # Filter by watcher/severity
|
|
34
|
+
clawdoctor install-service # Install as systemd user service
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## What It Monitors
|
|
38
|
+
|
|
39
|
+
| Monitor | What It Watches | Interval |
|
|
40
|
+
|---------|-----------------|----------|
|
|
41
|
+
| **GatewayWatcher** | `openclaw` process running | 30s |
|
|
42
|
+
| **CronWatcher** | `~/.openclaw/state/cron-*.json` for missed/failed crons | 60s |
|
|
43
|
+
| **SessionWatcher** | `~/.openclaw/agents/*/sessions/*.jsonl` for errors, aborts, stuck sessions | 60s |
|
|
44
|
+
| **AuthWatcher** | Gateway logs for 401/403/token expired patterns | 60s |
|
|
45
|
+
| **CostWatcher** | Session token costs — flags if >3x rolling average | 5m |
|
|
46
|
+
|
|
47
|
+
## What It Fixes
|
|
48
|
+
|
|
49
|
+
| Healer | Action |
|
|
50
|
+
|--------|--------|
|
|
51
|
+
| **ProcessHealer** | Restarts gateway via `systemctl restart openclaw-gateway` or `openclaw gateway restart`, then verifies |
|
|
52
|
+
| **CronHealer** | Logs the failure and includes the manual rerun command in the alert (Phase 0 — no auto-rerun) |
|
|
53
|
+
|
|
54
|
+
## Alerts
|
|
55
|
+
|
|
56
|
+
Telegram alerts with rate limiting (max 1 per monitor per 5 minutes):
|
|
57
|
+
|
|
58
|
+
```
|
|
59
|
+
🔴 ClawDoctor Alert
|
|
60
|
+
Monitor: GatewayWatcher
|
|
61
|
+
Event: Gateway process not found
|
|
62
|
+
Action: openclaw gateway restart
|
|
63
|
+
Status: ✅ Back online
|
|
64
|
+
─────
|
|
65
|
+
Time: 2026-03-15 03:14 UTC
|
|
66
|
+
Host: devbox
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## Configuration
|
|
70
|
+
|
|
71
|
+
Config lives at `~/.clawdoctor/config.json`:
|
|
72
|
+
|
|
73
|
+
```json
|
|
74
|
+
{
|
|
75
|
+
"openclawPath": "/root/.openclaw",
|
|
76
|
+
"watchers": {
|
|
77
|
+
"gateway": { "enabled": true, "interval": 30 },
|
|
78
|
+
"cron": { "enabled": true, "interval": 60 },
|
|
79
|
+
"session": { "enabled": true, "interval": 60 },
|
|
80
|
+
"auth": { "enabled": true, "interval": 60 },
|
|
81
|
+
"cost": { "enabled": true, "interval": 300 }
|
|
82
|
+
},
|
|
83
|
+
"healers": {
|
|
84
|
+
"processRestart": { "enabled": true },
|
|
85
|
+
"cronRetry": { "enabled": false }
|
|
86
|
+
},
|
|
87
|
+
"alerts": {
|
|
88
|
+
"telegram": {
|
|
89
|
+
"enabled": true,
|
|
90
|
+
"botToken": "your-bot-token",
|
|
91
|
+
"chatId": "your-chat-id"
|
|
92
|
+
}
|
|
93
|
+
},
|
|
94
|
+
"dryRun": false,
|
|
95
|
+
"retentionDays": 7
|
|
96
|
+
}
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Events are stored in `~/.clawdoctor/events.db` (SQLite) and retained for 7 days by default.
|
|
100
|
+
|
|
101
|
+
## Systemd
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
clawdoctor install-service
|
|
105
|
+
systemctl --user daemon-reload
|
|
106
|
+
systemctl --user enable clawdoctor
|
|
107
|
+
systemctl --user start clawdoctor
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
## License
|
|
111
|
+
|
|
112
|
+
MIT
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { ClawDoctorConfig } from '../config.js';
|
|
2
|
+
import { WatchResult } from '../watchers/base.js';
|
|
3
|
+
import { HealResult } from '../healers/base.js';
|
|
4
|
+
interface AlertPayload {
|
|
5
|
+
watcher: string;
|
|
6
|
+
result: WatchResult;
|
|
7
|
+
healResult?: HealResult;
|
|
8
|
+
}
|
|
9
|
+
export declare class TelegramAlerter {
|
|
10
|
+
private config;
|
|
11
|
+
private lastAlertTime;
|
|
12
|
+
constructor(config: ClawDoctorConfig);
|
|
13
|
+
private isRateLimited;
|
|
14
|
+
private markAlerted;
|
|
15
|
+
shouldAlert(result: WatchResult): boolean;
|
|
16
|
+
sendAlert(payload: AlertPayload): Promise<boolean>;
|
|
17
|
+
formatMessage(payload: AlertPayload): string;
|
|
18
|
+
sendRecovery(watcherName: string, message: string): Promise<boolean>;
|
|
19
|
+
}
|
|
20
|
+
export {};
|
|
21
|
+
//# sourceMappingURL=telegram.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"telegram.d.ts","sourceRoot":"","sources":["../../src/alerters/telegram.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAOhD,UAAU,YAAY;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,WAAW,CAAC;IACpB,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,aAAa,CAAkC;gBAE3C,MAAM,EAAE,gBAAgB;IAIpC,OAAO,CAAC,aAAa;IAKrB,OAAO,CAAC,WAAW;IAInB,WAAW,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO;IAInC,SAAS,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC;IAuCxD,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM;IAsBtC,YAAY,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;CAS3E"}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.TelegramAlerter = void 0;
|
|
4
|
+
const utils_js_1 = require("../utils.js");
|
|
5
|
+
const RATE_LIMIT_MS = 5 * 60 * 1000; // 5 minutes per monitor
|
|
6
|
+
const TELEGRAM_API = 'https://api.telegram.org';
|
|
7
|
+
class TelegramAlerter {
|
|
8
|
+
config;
|
|
9
|
+
lastAlertTime = new Map();
|
|
10
|
+
constructor(config) {
|
|
11
|
+
this.config = config;
|
|
12
|
+
}
|
|
13
|
+
isRateLimited(watcherName) {
|
|
14
|
+
const lastAlert = this.lastAlertTime.get(watcherName) ?? 0;
|
|
15
|
+
return Date.now() - lastAlert < RATE_LIMIT_MS;
|
|
16
|
+
}
|
|
17
|
+
markAlerted(watcherName) {
|
|
18
|
+
this.lastAlertTime.set(watcherName, Date.now());
|
|
19
|
+
}
|
|
20
|
+
shouldAlert(result) {
|
|
21
|
+
return result.severity === 'error' || result.severity === 'critical' || result.severity === 'warning';
|
|
22
|
+
}
|
|
23
|
+
async sendAlert(payload) {
|
|
24
|
+
const { telegram } = this.config.alerts;
|
|
25
|
+
if (!telegram.enabled || !telegram.botToken || !telegram.chatId) {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
if (this.isRateLimited(payload.watcher)) {
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
const message = this.formatMessage(payload);
|
|
32
|
+
try {
|
|
33
|
+
const url = `${TELEGRAM_API}/bot${telegram.botToken}/sendMessage`;
|
|
34
|
+
const response = await fetch(url, {
|
|
35
|
+
method: 'POST',
|
|
36
|
+
headers: { 'Content-Type': 'application/json' },
|
|
37
|
+
body: JSON.stringify({
|
|
38
|
+
chat_id: telegram.chatId,
|
|
39
|
+
text: message,
|
|
40
|
+
parse_mode: 'HTML',
|
|
41
|
+
}),
|
|
42
|
+
});
|
|
43
|
+
if (response.ok) {
|
|
44
|
+
this.markAlerted(payload.watcher);
|
|
45
|
+
return true;
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
const body = await response.text();
|
|
49
|
+
console.error(`[TelegramAlerter] Failed to send alert: ${response.status} ${body}`);
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
catch (err) {
|
|
54
|
+
console.error(`[TelegramAlerter] Error sending alert:`, err);
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
formatMessage(payload) {
|
|
59
|
+
const { watcher, result, healResult } = payload;
|
|
60
|
+
const isOk = result.ok || result.severity === 'info';
|
|
61
|
+
const icon = isOk ? '🟢' : (result.severity === 'critical' ? '🔴' : '🟡');
|
|
62
|
+
const actionLine = healResult
|
|
63
|
+
? `Action: ${healResult.action}\nStatus: ${healResult.success ? '✅ ' + healResult.message : '❌ ' + healResult.message}`
|
|
64
|
+
: '';
|
|
65
|
+
const lines = [
|
|
66
|
+
`${icon} <b>ClawDoctor Alert</b>`,
|
|
67
|
+
`Monitor: ${watcher}`,
|
|
68
|
+
`Event: ${result.message}`,
|
|
69
|
+
...(actionLine ? [actionLine] : []),
|
|
70
|
+
'─────',
|
|
71
|
+
`Time: ${(0, utils_js_1.nowUtcDisplay)()}`,
|
|
72
|
+
`Host: ${(0, utils_js_1.hostname)()}`,
|
|
73
|
+
];
|
|
74
|
+
return lines.join('\n');
|
|
75
|
+
}
|
|
76
|
+
async sendRecovery(watcherName, message) {
|
|
77
|
+
const fakeResult = {
|
|
78
|
+
ok: true,
|
|
79
|
+
severity: 'info',
|
|
80
|
+
event_type: 'recovered',
|
|
81
|
+
message,
|
|
82
|
+
};
|
|
83
|
+
return this.sendAlert({ watcher: watcherName, result: fakeResult });
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
exports.TelegramAlerter = TelegramAlerter;
|
|
87
|
+
//# sourceMappingURL=telegram.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"telegram.js","sourceRoot":"","sources":["../../src/alerters/telegram.ts"],"names":[],"mappings":";;;AAGA,0CAAsD;AAEtD,MAAM,aAAa,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,wBAAwB;AAE7D,MAAM,YAAY,GAAG,0BAA0B,CAAC;AAQhD,MAAa,eAAe;IAClB,MAAM,CAAmB;IACzB,aAAa,GAAwB,IAAI,GAAG,EAAE,CAAC;IAEvD,YAAY,MAAwB;QAClC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAEO,aAAa,CAAC,WAAmB;QACvC,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC3D,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,aAAa,CAAC;IAChD,CAAC;IAEO,WAAW,CAAC,WAAmB;QACrC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,WAAW,CAAC,MAAmB;QAC7B,OAAO,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,UAAU,IAAI,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC;IACxG,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,OAAqB;QACnC,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAExC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YAChE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACxC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAE5C,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,GAAG,YAAY,OAAO,QAAQ,CAAC,QAAQ,cAAc,CAAC;YAClE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAChC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,QAAQ,CAAC,MAAM;oBACxB,IAAI,EAAE,OAAO;oBACb,UAAU,EAAE,MAAM;iBACnB,CAAC;aACH,CAAC,CAAC;YAEH,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;gBAChB,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBAClC,OAAO,IAAI,CAAC;YACd,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnC,OAAO,CAAC,KAAK,CAAC,2CAA2C,QAAQ,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC,CAAC;gBACpF,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,wCAAwC,EAAE,GAAG,CAAC,CAAC;YAC7D,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,aAAa,CAAC,OAAqB;QACjC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;QAChD,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,IAAI,MAAM,CAAC,QAAQ,KAAK,MAAM,CAAC;QACrD,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAE1E,MAAM,UAAU,GAAG,UAAU;YAC3B,CAAC,CAAC,WAAW,UAAU,CAAC,MAAM,aAAa,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,GAAG,UAAU,CAAC,OAAO,EAAE;YACvH,CAAC,CAAC,EAAE,CAAC;QAEP,MAAM,KAAK,GAAG;YACZ,GAAG,IAAI,0BAA0B;YACjC,YAAY,OAAO,EAAE;YACrB,UAAU,MAAM,CAAC,OAAO,EAAE;YAC1B,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACnC,OAAO;YACP,SAAS,IAAA,wBAAa,GAAE,EAAE;YAC1B,SAAS,IAAA,mBAAQ,GAAE,EAAE;SACtB,CAAC;QAEF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,WAAmB,EAAE,OAAe;QACrD,MAAM,UAAU,GAAgB;YAC9B,EAAE,EAAE,IAAI;YACR,QAAQ,EAAE,MAAM;YAChB,UAAU,EAAE,WAAW;YACvB,OAAO;SACR,CAAC;QACF,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IACtE,CAAC;CACF;AA3FD,0CA2FC"}
|
package/dist/config.d.ts
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
export interface WatcherConfig {
|
|
2
|
+
enabled: boolean;
|
|
3
|
+
interval: number;
|
|
4
|
+
}
|
|
5
|
+
export interface HealerConfig {
|
|
6
|
+
enabled: boolean;
|
|
7
|
+
}
|
|
8
|
+
export interface TelegramConfig {
|
|
9
|
+
enabled: boolean;
|
|
10
|
+
botToken: string;
|
|
11
|
+
chatId: string;
|
|
12
|
+
}
|
|
13
|
+
export interface AlertsConfig {
|
|
14
|
+
telegram: TelegramConfig;
|
|
15
|
+
}
|
|
16
|
+
export interface ClawDoctorConfig {
|
|
17
|
+
openclawPath: string;
|
|
18
|
+
watchers: {
|
|
19
|
+
gateway: WatcherConfig;
|
|
20
|
+
cron: WatcherConfig;
|
|
21
|
+
session: WatcherConfig;
|
|
22
|
+
auth: WatcherConfig;
|
|
23
|
+
cost: WatcherConfig;
|
|
24
|
+
};
|
|
25
|
+
healers: {
|
|
26
|
+
processRestart: HealerConfig;
|
|
27
|
+
cronRetry: HealerConfig;
|
|
28
|
+
};
|
|
29
|
+
alerts: AlertsConfig;
|
|
30
|
+
dryRun: boolean;
|
|
31
|
+
retentionDays: number;
|
|
32
|
+
}
|
|
33
|
+
export declare const AGENTWATCH_DIR: string;
|
|
34
|
+
export declare const CONFIG_PATH: string;
|
|
35
|
+
export declare const DB_PATH: string;
|
|
36
|
+
export declare const PID_PATH: string;
|
|
37
|
+
export declare const DEFAULT_CONFIG: ClawDoctorConfig;
|
|
38
|
+
export declare function loadConfig(): ClawDoctorConfig;
|
|
39
|
+
export declare function saveConfig(config: ClawDoctorConfig): void;
|
|
40
|
+
export declare function ensureAgentwatchDir(): void;
|
|
41
|
+
export declare function configExists(): boolean;
|
|
42
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,cAAc,CAAC;CAC1B;AAED,MAAM,WAAW,gBAAgB;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE;QACR,OAAO,EAAE,aAAa,CAAC;QACvB,IAAI,EAAE,aAAa,CAAC;QACpB,OAAO,EAAE,aAAa,CAAC;QACvB,IAAI,EAAE,aAAa,CAAC;QACpB,IAAI,EAAE,aAAa,CAAC;KACrB,CAAC;IACF,OAAO,EAAE;QACP,cAAc,EAAE,YAAY,CAAC;QAC7B,SAAS,EAAE,YAAY,CAAC;KACzB,CAAC;IACF,MAAM,EAAE,YAAY,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,eAAO,MAAM,cAAc,QAAyC,CAAC;AACrE,eAAO,MAAM,WAAW,QAA2C,CAAC;AACpE,eAAO,MAAM,OAAO,QAAyC,CAAC;AAC9D,eAAO,MAAM,QAAQ,QAA8C,CAAC;AAEpE,eAAO,MAAM,cAAc,EAAE,gBAsB5B,CAAC;AAEF,wBAAgB,UAAU,IAAI,gBAAgB,CAO7C;AAmBD,wBAAgB,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAGzD;AAED,wBAAgB,mBAAmB,IAAI,IAAI,CAI1C;AAED,wBAAgB,YAAY,IAAI,OAAO,CAEtC"}
|
package/dist/config.js
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.DEFAULT_CONFIG = exports.PID_PATH = exports.DB_PATH = exports.CONFIG_PATH = exports.AGENTWATCH_DIR = void 0;
|
|
7
|
+
exports.loadConfig = loadConfig;
|
|
8
|
+
exports.saveConfig = saveConfig;
|
|
9
|
+
exports.ensureAgentwatchDir = ensureAgentwatchDir;
|
|
10
|
+
exports.configExists = configExists;
|
|
11
|
+
const fs_1 = __importDefault(require("fs"));
|
|
12
|
+
const path_1 = __importDefault(require("path"));
|
|
13
|
+
const os_1 = __importDefault(require("os"));
|
|
14
|
+
exports.AGENTWATCH_DIR = path_1.default.join(os_1.default.homedir(), '.clawdoctor');
|
|
15
|
+
exports.CONFIG_PATH = path_1.default.join(exports.AGENTWATCH_DIR, 'config.json');
|
|
16
|
+
exports.DB_PATH = path_1.default.join(exports.AGENTWATCH_DIR, 'events.db');
|
|
17
|
+
exports.PID_PATH = path_1.default.join(exports.AGENTWATCH_DIR, 'clawdoctor.pid');
|
|
18
|
+
exports.DEFAULT_CONFIG = {
|
|
19
|
+
openclawPath: path_1.default.join(os_1.default.homedir(), '.openclaw'),
|
|
20
|
+
watchers: {
|
|
21
|
+
gateway: { enabled: true, interval: 30 },
|
|
22
|
+
cron: { enabled: true, interval: 60 },
|
|
23
|
+
session: { enabled: true, interval: 60 },
|
|
24
|
+
auth: { enabled: true, interval: 60 },
|
|
25
|
+
cost: { enabled: true, interval: 300 },
|
|
26
|
+
},
|
|
27
|
+
healers: {
|
|
28
|
+
processRestart: { enabled: true },
|
|
29
|
+
cronRetry: { enabled: false },
|
|
30
|
+
},
|
|
31
|
+
alerts: {
|
|
32
|
+
telegram: {
|
|
33
|
+
enabled: false,
|
|
34
|
+
botToken: '',
|
|
35
|
+
chatId: '',
|
|
36
|
+
},
|
|
37
|
+
},
|
|
38
|
+
dryRun: false,
|
|
39
|
+
retentionDays: 7,
|
|
40
|
+
};
|
|
41
|
+
function loadConfig() {
|
|
42
|
+
if (!fs_1.default.existsSync(exports.CONFIG_PATH)) {
|
|
43
|
+
throw new Error(`Config not found at ${exports.CONFIG_PATH}. Run 'clawdoctor init' first.`);
|
|
44
|
+
}
|
|
45
|
+
const raw = fs_1.default.readFileSync(exports.CONFIG_PATH, 'utf-8');
|
|
46
|
+
const parsed = JSON.parse(raw);
|
|
47
|
+
return mergeConfig(exports.DEFAULT_CONFIG, parsed);
|
|
48
|
+
}
|
|
49
|
+
function mergeConfig(defaults, overrides) {
|
|
50
|
+
return {
|
|
51
|
+
...defaults,
|
|
52
|
+
...overrides,
|
|
53
|
+
watchers: { ...defaults.watchers, ...(overrides.watchers ?? {}) },
|
|
54
|
+
healers: { ...defaults.healers, ...(overrides.healers ?? {}) },
|
|
55
|
+
alerts: {
|
|
56
|
+
...defaults.alerts,
|
|
57
|
+
...(overrides.alerts ?? {}),
|
|
58
|
+
telegram: {
|
|
59
|
+
...defaults.alerts.telegram,
|
|
60
|
+
...(overrides.alerts?.telegram ?? {}),
|
|
61
|
+
},
|
|
62
|
+
},
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
function saveConfig(config) {
|
|
66
|
+
ensureAgentwatchDir();
|
|
67
|
+
fs_1.default.writeFileSync(exports.CONFIG_PATH, JSON.stringify(config, null, 2), 'utf-8');
|
|
68
|
+
}
|
|
69
|
+
function ensureAgentwatchDir() {
|
|
70
|
+
if (!fs_1.default.existsSync(exports.AGENTWATCH_DIR)) {
|
|
71
|
+
fs_1.default.mkdirSync(exports.AGENTWATCH_DIR, { recursive: true });
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
function configExists() {
|
|
75
|
+
return fs_1.default.existsSync(exports.CONFIG_PATH);
|
|
76
|
+
}
|
|
77
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";;;;;;AAsEA,gCAOC;AAmBD,gCAGC;AAED,kDAIC;AAED,oCAEC;AA7GD,4CAAoB;AACpB,gDAAwB;AACxB,4CAAoB;AAuCP,QAAA,cAAc,GAAG,cAAI,CAAC,IAAI,CAAC,YAAE,CAAC,OAAO,EAAE,EAAE,aAAa,CAAC,CAAC;AACxD,QAAA,WAAW,GAAG,cAAI,CAAC,IAAI,CAAC,sBAAc,EAAE,aAAa,CAAC,CAAC;AACvD,QAAA,OAAO,GAAG,cAAI,CAAC,IAAI,CAAC,sBAAc,EAAE,WAAW,CAAC,CAAC;AACjD,QAAA,QAAQ,GAAG,cAAI,CAAC,IAAI,CAAC,sBAAc,EAAE,gBAAgB,CAAC,CAAC;AAEvD,QAAA,cAAc,GAAqB;IAC9C,YAAY,EAAE,cAAI,CAAC,IAAI,CAAC,YAAE,CAAC,OAAO,EAAE,EAAE,WAAW,CAAC;IAClD,QAAQ,EAAE;QACR,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE;QACxC,IAAI,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE;QACrC,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE;QACxC,IAAI,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE;QACrC,IAAI,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE;KACvC;IACD,OAAO,EAAE;QACP,cAAc,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;QACjC,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE;KAC9B;IACD,MAAM,EAAE;QACN,QAAQ,EAAE;YACR,OAAO,EAAE,KAAK;YACd,QAAQ,EAAE,EAAE;YACZ,MAAM,EAAE,EAAE;SACX;KACF;IACD,MAAM,EAAE,KAAK;IACb,aAAa,EAAE,CAAC;CACjB,CAAC;AAEF,SAAgB,UAAU;IACxB,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,mBAAW,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CAAC,uBAAuB,mBAAW,gCAAgC,CAAC,CAAC;IACtF,CAAC;IACD,MAAM,GAAG,GAAG,YAAE,CAAC,YAAY,CAAC,mBAAW,EAAE,OAAO,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAA8B,CAAC;IAC5D,OAAO,WAAW,CAAC,sBAAc,EAAE,MAAM,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,WAAW,CAAC,QAA0B,EAAE,SAAoC;IACnF,OAAO;QACL,GAAG,QAAQ;QACX,GAAG,SAAS;QACZ,QAAQ,EAAE,EAAE,GAAG,QAAQ,CAAC,QAAQ,EAAE,GAAG,CAAC,SAAS,CAAC,QAAQ,IAAI,EAAE,CAAC,EAAE;QACjE,OAAO,EAAE,EAAE,GAAG,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE;QAC9D,MAAM,EAAE;YACN,GAAG,QAAQ,CAAC,MAAM;YAClB,GAAG,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,CAAC;YAC3B,QAAQ,EAAE;gBACR,GAAG,QAAQ,CAAC,MAAM,CAAC,QAAQ;gBAC3B,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,IAAI,EAAE,CAAC;aACtC;SACF;KACF,CAAC;AACJ,CAAC;AAED,SAAgB,UAAU,CAAC,MAAwB;IACjD,mBAAmB,EAAE,CAAC;IACtB,YAAE,CAAC,aAAa,CAAC,mBAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AAC1E,CAAC;AAED,SAAgB,mBAAmB;IACjC,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,sBAAc,CAAC,EAAE,CAAC;QACnC,YAAE,CAAC,SAAS,CAAC,sBAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpD,CAAC;AACH,CAAC;AAED,SAAgB,YAAY;IAC1B,OAAO,YAAE,CAAC,UAAU,CAAC,mBAAW,CAAC,CAAC;AACpC,CAAC"}
|
package/dist/daemon.d.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { ClawDoctorConfig } from './config.js';
|
|
2
|
+
import { WatchResult } from './watchers/base.js';
|
|
3
|
+
export declare class Daemon {
|
|
4
|
+
private config;
|
|
5
|
+
private watchers;
|
|
6
|
+
private alerter;
|
|
7
|
+
private processHealer;
|
|
8
|
+
private cronHealer;
|
|
9
|
+
private running;
|
|
10
|
+
private tickInterval;
|
|
11
|
+
constructor(config: ClawDoctorConfig);
|
|
12
|
+
private setupWatchers;
|
|
13
|
+
start(): void;
|
|
14
|
+
stop(): void;
|
|
15
|
+
private tick;
|
|
16
|
+
private runWatcher;
|
|
17
|
+
private handleResult;
|
|
18
|
+
private attemptHeal;
|
|
19
|
+
runOnce(): Promise<Map<string, WatchResult[]>>;
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=daemon.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"daemon.d.ts","sourceRoot":"","sources":["../src/daemon.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAe,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAkB9D,qBAAa,MAAM;IACjB,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,QAAQ,CAAsB;IACtC,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,aAAa,CAAgB;IACrC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,YAAY,CAA+B;gBAEvC,MAAM,EAAE,gBAAgB;IAQpC,OAAO,CAAC,aAAa;IAwCrB,KAAK,IAAI,IAAI;IA0Bb,IAAI,IAAI,IAAI;IAUZ,OAAO,CAAC,IAAI;YAYE,UAAU;YAWV,YAAY;YAkBZ,WAAW;IAsBnB,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;CAiBrD"}
|
package/dist/daemon.js
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Daemon = void 0;
|
|
4
|
+
const gateway_js_1 = require("./watchers/gateway.js");
|
|
5
|
+
const cron_js_1 = require("./watchers/cron.js");
|
|
6
|
+
const session_js_1 = require("./watchers/session.js");
|
|
7
|
+
const auth_js_1 = require("./watchers/auth.js");
|
|
8
|
+
const cost_js_1 = require("./watchers/cost.js");
|
|
9
|
+
const process_js_1 = require("./healers/process.js");
|
|
10
|
+
const cron_js_2 = require("./healers/cron.js");
|
|
11
|
+
const telegram_js_1 = require("./alerters/telegram.js");
|
|
12
|
+
const store_js_1 = require("./store.js");
|
|
13
|
+
const utils_js_1 = require("./utils.js");
|
|
14
|
+
class Daemon {
|
|
15
|
+
config;
|
|
16
|
+
watchers = [];
|
|
17
|
+
alerter;
|
|
18
|
+
processHealer;
|
|
19
|
+
cronHealer;
|
|
20
|
+
running = false;
|
|
21
|
+
tickInterval = null;
|
|
22
|
+
constructor(config) {
|
|
23
|
+
this.config = config;
|
|
24
|
+
this.alerter = new telegram_js_1.TelegramAlerter(config);
|
|
25
|
+
this.processHealer = new process_js_1.ProcessHealer(config);
|
|
26
|
+
this.cronHealer = new cron_js_2.CronHealer(config);
|
|
27
|
+
this.setupWatchers();
|
|
28
|
+
}
|
|
29
|
+
setupWatchers() {
|
|
30
|
+
const { watchers } = this.config;
|
|
31
|
+
if (watchers.gateway.enabled) {
|
|
32
|
+
this.watchers.push({
|
|
33
|
+
watcher: new gateway_js_1.GatewayWatcher(this.config),
|
|
34
|
+
intervalMs: watchers.gateway.interval * 1000,
|
|
35
|
+
lastRun: 0,
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
if (watchers.cron.enabled) {
|
|
39
|
+
this.watchers.push({
|
|
40
|
+
watcher: new cron_js_1.CronWatcher(this.config),
|
|
41
|
+
intervalMs: watchers.cron.interval * 1000,
|
|
42
|
+
lastRun: 0,
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
if (watchers.session.enabled) {
|
|
46
|
+
this.watchers.push({
|
|
47
|
+
watcher: new session_js_1.SessionWatcher(this.config),
|
|
48
|
+
intervalMs: watchers.session.interval * 1000,
|
|
49
|
+
lastRun: 0,
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
if (watchers.auth.enabled) {
|
|
53
|
+
this.watchers.push({
|
|
54
|
+
watcher: new auth_js_1.AuthWatcher(this.config),
|
|
55
|
+
intervalMs: watchers.auth.interval * 1000,
|
|
56
|
+
lastRun: 0,
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
if (watchers.cost.enabled) {
|
|
60
|
+
this.watchers.push({
|
|
61
|
+
watcher: new cost_js_1.CostWatcher(this.config),
|
|
62
|
+
intervalMs: watchers.cost.interval * 1000,
|
|
63
|
+
lastRun: 0,
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
start() {
|
|
68
|
+
if (this.running)
|
|
69
|
+
return;
|
|
70
|
+
this.running = true;
|
|
71
|
+
console.log(`[${(0, utils_js_1.nowIso)()}] ClawDoctor daemon started`);
|
|
72
|
+
console.log(`[${(0, utils_js_1.nowIso)()}] Monitoring ${this.watchers.length} watcher(s)`);
|
|
73
|
+
if (this.config.dryRun) {
|
|
74
|
+
console.log(`[${(0, utils_js_1.nowIso)()}] DRY RUN mode — healers will not take action`);
|
|
75
|
+
}
|
|
76
|
+
// Run all watchers immediately on start
|
|
77
|
+
this.tick();
|
|
78
|
+
// Tick every 5 seconds to check if any watcher is due
|
|
79
|
+
this.tickInterval = setInterval(() => this.tick(), 5000);
|
|
80
|
+
// Prune old events daily
|
|
81
|
+
setInterval(() => {
|
|
82
|
+
const pruned = (0, store_js_1.pruneOldEvents)(this.config.retentionDays);
|
|
83
|
+
if (pruned > 0) {
|
|
84
|
+
console.log(`[${(0, utils_js_1.nowIso)()}] Pruned ${pruned} old event(s)`);
|
|
85
|
+
}
|
|
86
|
+
}, 24 * 3600 * 1000);
|
|
87
|
+
}
|
|
88
|
+
stop() {
|
|
89
|
+
if (!this.running)
|
|
90
|
+
return;
|
|
91
|
+
this.running = false;
|
|
92
|
+
if (this.tickInterval) {
|
|
93
|
+
clearInterval(this.tickInterval);
|
|
94
|
+
this.tickInterval = null;
|
|
95
|
+
}
|
|
96
|
+
console.log(`[${(0, utils_js_1.nowIso)()}] ClawDoctor daemon stopped`);
|
|
97
|
+
}
|
|
98
|
+
tick() {
|
|
99
|
+
const now = Date.now();
|
|
100
|
+
for (const entry of this.watchers) {
|
|
101
|
+
if (now - entry.lastRun >= entry.intervalMs) {
|
|
102
|
+
entry.lastRun = now;
|
|
103
|
+
this.runWatcher(entry.watcher).catch(err => {
|
|
104
|
+
console.error(`[${(0, utils_js_1.nowIso)()}] Error in ${entry.watcher.name}:`, err);
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
async runWatcher(watcher) {
|
|
110
|
+
try {
|
|
111
|
+
const results = await watcher.run();
|
|
112
|
+
for (const result of results) {
|
|
113
|
+
await this.handleResult(watcher, result);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
catch (err) {
|
|
117
|
+
console.error(`[${(0, utils_js_1.nowIso)()}] ${watcher.name} threw:`, err);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
async handleResult(watcher, result) {
|
|
121
|
+
const prefix = `[${(0, utils_js_1.nowIso)()}] [${watcher.name}]`;
|
|
122
|
+
if (result.severity !== 'info') {
|
|
123
|
+
console.log(`${prefix} ${result.severity.toUpperCase()}: ${result.message}`);
|
|
124
|
+
}
|
|
125
|
+
// Auto-healing
|
|
126
|
+
if (!result.ok) {
|
|
127
|
+
const healResult = await this.attemptHeal(watcher, result);
|
|
128
|
+
if (healResult && this.alerter.shouldAlert(result)) {
|
|
129
|
+
await this.alerter.sendAlert({ watcher: watcher.name, result, healResult });
|
|
130
|
+
}
|
|
131
|
+
else if (this.alerter.shouldAlert(result)) {
|
|
132
|
+
await this.alerter.sendAlert({ watcher: watcher.name, result });
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
async attemptHeal(watcher, result) {
|
|
137
|
+
if (watcher.name === 'GatewayWatcher' && result.event_type === 'gateway_down') {
|
|
138
|
+
if (this.config.healers.processRestart.enabled) {
|
|
139
|
+
console.log(`[${(0, utils_js_1.nowIso)()}] [ProcessHealer] Attempting to restart gateway...`);
|
|
140
|
+
const healResult = await this.processHealer.heal({});
|
|
141
|
+
console.log(`[${(0, utils_js_1.nowIso)()}] [ProcessHealer] ${healResult.success ? 'SUCCESS' : 'FAILED'}: ${healResult.message}`);
|
|
142
|
+
return healResult;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
if (watcher.name === 'CronWatcher' && (result.event_type === 'cron_error' || result.event_type === 'cron_overdue')) {
|
|
146
|
+
const context = result.details ?? {};
|
|
147
|
+
const healResult = await this.cronHealer.heal(context);
|
|
148
|
+
return healResult;
|
|
149
|
+
}
|
|
150
|
+
return null;
|
|
151
|
+
}
|
|
152
|
+
async runOnce() {
|
|
153
|
+
const allResults = new Map();
|
|
154
|
+
for (const entry of this.watchers) {
|
|
155
|
+
try {
|
|
156
|
+
const results = await entry.watcher.check();
|
|
157
|
+
allResults.set(entry.watcher.name, results);
|
|
158
|
+
}
|
|
159
|
+
catch (err) {
|
|
160
|
+
allResults.set(entry.watcher.name, [{
|
|
161
|
+
ok: false,
|
|
162
|
+
severity: 'error',
|
|
163
|
+
event_type: 'watcher_error',
|
|
164
|
+
message: `Watcher threw: ${String(err)}`,
|
|
165
|
+
}]);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return allResults;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
exports.Daemon = Daemon;
|
|
172
|
+
//# sourceMappingURL=daemon.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"daemon.js","sourceRoot":"","sources":["../src/daemon.ts"],"names":[],"mappings":";;;AAEA,sDAAuD;AACvD,gDAAiD;AACjD,sDAAuD;AACvD,gDAAiD;AACjD,gDAAiD;AACjD,qDAAqD;AACrD,+CAA+C;AAC/C,wDAAyD;AACzD,yCAA4C;AAC5C,yCAAoC;AAQpC,MAAa,MAAM;IACT,MAAM,CAAmB;IACzB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,OAAO,CAAkB;IACzB,aAAa,CAAgB;IAC7B,UAAU,CAAa;IACvB,OAAO,GAAG,KAAK,CAAC;IAChB,YAAY,GAA0B,IAAI,CAAC;IAEnD,YAAY,MAAwB;QAClC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,IAAI,6BAAe,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,CAAC,aAAa,GAAG,IAAI,0BAAa,CAAC,MAAM,CAAC,CAAC;QAC/C,IAAI,CAAC,UAAU,GAAG,IAAI,oBAAU,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC;IAEO,aAAa;QACnB,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAEjC,IAAI,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACjB,OAAO,EAAE,IAAI,2BAAc,CAAC,IAAI,CAAC,MAAM,CAAC;gBACxC,UAAU,EAAE,QAAQ,CAAC,OAAO,CAAC,QAAQ,GAAG,IAAI;gBAC5C,OAAO,EAAE,CAAC;aACX,CAAC,CAAC;QACL,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACjB,OAAO,EAAE,IAAI,qBAAW,CAAC,IAAI,CAAC,MAAM,CAAC;gBACrC,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI;gBACzC,OAAO,EAAE,CAAC;aACX,CAAC,CAAC;QACL,CAAC;QACD,IAAI,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACjB,OAAO,EAAE,IAAI,2BAAc,CAAC,IAAI,CAAC,MAAM,CAAC;gBACxC,UAAU,EAAE,QAAQ,CAAC,OAAO,CAAC,QAAQ,GAAG,IAAI;gBAC5C,OAAO,EAAE,CAAC;aACX,CAAC,CAAC;QACL,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACjB,OAAO,EAAE,IAAI,qBAAW,CAAC,IAAI,CAAC,MAAM,CAAC;gBACrC,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI;gBACzC,OAAO,EAAE,CAAC;aACX,CAAC,CAAC;QACL,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACjB,OAAO,EAAE,IAAI,qBAAW,CAAC,IAAI,CAAC,MAAM,CAAC;gBACrC,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI;gBACzC,OAAO,EAAE,CAAC;aACX,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QAEpB,OAAO,CAAC,GAAG,CAAC,IAAI,IAAA,iBAAM,GAAE,6BAA6B,CAAC,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,IAAI,IAAA,iBAAM,GAAE,gBAAgB,IAAI,CAAC,QAAQ,CAAC,MAAM,aAAa,CAAC,CAAC;QAE3E,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,IAAI,IAAA,iBAAM,GAAE,+CAA+C,CAAC,CAAC;QAC3E,CAAC;QAED,wCAAwC;QACxC,IAAI,CAAC,IAAI,EAAE,CAAC;QAEZ,sDAAsD;QACtD,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC;QAEzD,yBAAyB;QACzB,WAAW,CAAC,GAAG,EAAE;YACf,MAAM,MAAM,GAAG,IAAA,yBAAc,EAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACzD,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,OAAO,CAAC,GAAG,CAAC,IAAI,IAAA,iBAAM,GAAE,YAAY,MAAM,eAAe,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;IACvB,CAAC;IAED,IAAI;QACF,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO;QAC1B,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACrB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACjC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC3B,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,IAAI,IAAA,iBAAM,GAAE,6BAA6B,CAAC,CAAC;IACzD,CAAC;IAEO,IAAI;QACV,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,IAAI,GAAG,GAAG,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;gBAC5C,KAAK,CAAC,OAAO,GAAG,GAAG,CAAC;gBACpB,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE;oBACzC,OAAO,CAAC,KAAK,CAAC,IAAI,IAAA,iBAAM,GAAE,cAAc,KAAK,CAAC,OAAO,CAAC,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;gBACtE,CAAC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,OAAoB;QAC3C,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC;YACpC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,IAAI,IAAA,iBAAM,GAAE,KAAK,OAAO,CAAC,IAAI,SAAS,EAAE,GAAG,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,OAAoB,EAAE,MAAmB;QAClE,MAAM,MAAM,GAAG,IAAI,IAAA,iBAAM,GAAE,MAAM,OAAO,CAAC,IAAI,GAAG,CAAC;QAEjD,IAAI,MAAM,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/E,CAAC;QAED,eAAe;QACf,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3D,IAAI,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;gBACnD,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;YAC9E,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5C,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;YAClE,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,OAAoB,EACpB,MAAmB;QAEnB,IAAI,OAAO,CAAC,IAAI,KAAK,gBAAgB,IAAI,MAAM,CAAC,UAAU,KAAK,cAAc,EAAE,CAAC;YAC9E,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;gBAC/C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAA,iBAAM,GAAE,oDAAoD,CAAC,CAAC;gBAC9E,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACrD,OAAO,CAAC,GAAG,CAAC,IAAI,IAAA,iBAAM,GAAE,qBAAqB,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,KAAK,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;gBACjH,OAAO,UAAU,CAAC;YACpB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,aAAa,IAAI,CAAC,MAAM,CAAC,UAAU,KAAK,YAAY,IAAI,MAAM,CAAC,UAAU,KAAK,cAAc,CAAC,EAAE,CAAC;YACnH,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;YACrC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAkC,CAAC,CAAC;YAClF,OAAO,UAAU,CAAC;QACpB,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,UAAU,GAAG,IAAI,GAAG,EAAyB,CAAC;QACpD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBAC5C,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC9C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;wBAClC,EAAE,EAAE,KAAK;wBACT,QAAQ,EAAE,OAAO;wBACjB,UAAU,EAAE,eAAe;wBAC3B,OAAO,EAAE,kBAAkB,MAAM,CAAC,GAAG,CAAC,EAAE;qBACzC,CAAC,CAAC,CAAC;YACN,CAAC;QACH,CAAC;QACD,OAAO,UAAU,CAAC;IACpB,CAAC;CACF;AA7KD,wBA6KC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { ClawDoctorConfig } from '../config.js';
|
|
2
|
+
export interface HealResult {
|
|
3
|
+
success: boolean;
|
|
4
|
+
action: string;
|
|
5
|
+
message: string;
|
|
6
|
+
details?: Record<string, unknown>;
|
|
7
|
+
}
|
|
8
|
+
export declare abstract class BaseHealer {
|
|
9
|
+
abstract readonly name: string;
|
|
10
|
+
protected config: ClawDoctorConfig;
|
|
11
|
+
constructor(config: ClawDoctorConfig);
|
|
12
|
+
abstract heal(context: Record<string, unknown>): Promise<HealResult>;
|
|
13
|
+
protected recordHeal(watcherName: string, result: HealResult, eventType: string): Promise<void>;
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=base.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../src/healers/base.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAIhD,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED,8BAAsB,UAAU;IAC9B,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAE/B,SAAS,CAAC,MAAM,EAAE,gBAAgB,CAAC;gBAEvB,MAAM,EAAE,gBAAgB;IAIpC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC;cAEpD,UAAU,CACxB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,UAAU,EAClB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC;CAYjB"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.BaseHealer = void 0;
|
|
4
|
+
const store_js_1 = require("../store.js");
|
|
5
|
+
const utils_js_1 = require("../utils.js");
|
|
6
|
+
class BaseHealer {
|
|
7
|
+
config;
|
|
8
|
+
constructor(config) {
|
|
9
|
+
this.config = config;
|
|
10
|
+
}
|
|
11
|
+
async recordHeal(watcherName, result, eventType) {
|
|
12
|
+
(0, store_js_1.insertEvent)({
|
|
13
|
+
timestamp: (0, utils_js_1.nowIso)(),
|
|
14
|
+
watcher: watcherName,
|
|
15
|
+
severity: result.success ? 'info' : 'error',
|
|
16
|
+
event_type: eventType,
|
|
17
|
+
message: result.message,
|
|
18
|
+
details: result.details ? JSON.stringify(result.details) : undefined,
|
|
19
|
+
action_taken: result.action,
|
|
20
|
+
action_result: result.success ? 'success' : 'failed',
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
exports.BaseHealer = BaseHealer;
|
|
25
|
+
//# sourceMappingURL=base.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"base.js","sourceRoot":"","sources":["../../src/healers/base.ts"],"names":[],"mappings":";;;AACA,0CAA0C;AAC1C,0CAAqC;AASrC,MAAsB,UAAU;IAGpB,MAAM,CAAmB;IAEnC,YAAY,MAAwB;QAClC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAIS,KAAK,CAAC,UAAU,CACxB,WAAmB,EACnB,MAAkB,EAClB,SAAiB;QAEjB,IAAA,sBAAW,EAAC;YACV,SAAS,EAAE,IAAA,iBAAM,GAAE;YACnB,OAAO,EAAE,WAAW;YACpB,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO;YAC3C,UAAU,EAAE,SAAS;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;YACpE,YAAY,EAAE,MAAM,CAAC,MAAM;YAC3B,aAAa,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ;SACrD,CAAC,CAAC;IACL,CAAC;CACF;AA3BD,gCA2BC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cron.d.ts","sourceRoot":"","sources":["../../src/healers/cron.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEnD,qBAAa,UAAW,SAAQ,UAAU;IACxC,QAAQ,CAAC,IAAI,gBAAgB;IAEvB,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC;CAoBlE"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.CronHealer = void 0;
|
|
4
|
+
const base_js_1 = require("./base.js");
|
|
5
|
+
class CronHealer extends base_js_1.BaseHealer {
|
|
6
|
+
name = 'CronHealer';
|
|
7
|
+
async heal(context) {
|
|
8
|
+
const cronName = context.cronName ?? 'unknown';
|
|
9
|
+
const lastRun = context.lastRun ?? 'unknown';
|
|
10
|
+
const lastError = context.lastError ?? undefined;
|
|
11
|
+
// Phase 0: do not auto-rerun crons, just log and suggest
|
|
12
|
+
const manualCommand = cronName !== 'unknown'
|
|
13
|
+
? `openclaw cron run ${cronName}`
|
|
14
|
+
: 'openclaw cron run <cron-name>';
|
|
15
|
+
const result = {
|
|
16
|
+
success: true,
|
|
17
|
+
action: `logged cron failure for '${cronName}'`,
|
|
18
|
+
message: `Cron '${cronName}' failed${lastError ? `: ${lastError}` : ''}. Last run: ${lastRun}. Manual rerun: \`${manualCommand}\``,
|
|
19
|
+
details: { cronName, lastRun, lastError, manualCommand },
|
|
20
|
+
};
|
|
21
|
+
await this.recordHeal('CronWatcher', result, 'cron_failure_logged');
|
|
22
|
+
return result;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
exports.CronHealer = CronHealer;
|
|
26
|
+
//# sourceMappingURL=cron.js.map
|