@flame0510/project-aether 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/agent-templates/README.md +42 -22
- package/agent-templates/base-image/Dockerfile +42 -33
- package/agent-templates/base-image/entrypoint.sh +67 -12
- package/app/agents/BrowserAccessSection.tsx +510 -0
- package/app/agents/ChannelManager.tsx +19 -11
- package/app/agents/ImageDownloadBanner.tsx +53 -19
- package/app/agents/ModelSection.tsx +4 -1
- package/app/agents/PageClient.tsx +629 -167
- package/app/agents/UpdateSection.tsx +300 -0
- package/app/agents/create/PageClient.tsx +11 -49
- package/app/api/agents/[id]/backup/route.ts +26 -69
- package/app/api/agents/[id]/channels/pairing/route.ts +3 -3
- package/app/api/agents/[id]/channels/telegram/route.ts +2 -2
- package/app/api/agents/[id]/cold-backup/route.ts +56 -0
- package/app/api/agents/[id]/devices/route.ts +126 -0
- package/app/api/agents/[id]/invite-link/route.ts +53 -0
- package/app/api/agents/[id]/lifecycle/route.ts +3 -0
- package/app/api/agents/[id]/open-control-ui/route.ts +58 -0
- package/app/api/agents/[id]/recreate/route.ts +33 -163
- package/app/api/agents/[id]/restart/route.ts +5 -0
- package/app/api/agents/[id]/restore/route.ts +40 -70
- package/app/api/agents/[id]/route.ts +38 -150
- package/app/api/agents/[id]/update/rollback/route.ts +30 -0
- package/app/api/agents/[id]/update/route.ts +50 -0
- package/app/api/agents/activity-summary/route.ts +67 -0
- package/app/api/agents/create/route.ts +32 -88
- package/app/api/agents/devices-summary/route.ts +37 -0
- package/app/api/agents/download-image/route.ts +16 -9
- package/app/api/agents/image-status/route.ts +31 -111
- package/app/api/agents/route.ts +25 -49
- package/app/api/agents/token/route.ts +33 -10
- package/app/api/assistant/route.ts +2 -2
- package/app/api/gateway/agent/route.ts +14 -0
- package/app/api/gateway/provider/balance/route.ts +5 -2
- package/app/api/gateway/sync.ts +97 -14
- package/app/api/setup/agent-image/route.ts +14 -42
- package/app/components/DashboardToolbar.tsx +1 -1
- package/app/gateway/PageClient.tsx +27 -32
- package/bin/rev4a.js +43 -41
- package/daemon.js +6 -6
- package/docs/ARCHITECTURE.md +95 -9
- package/docs/FRONTEND-ARCHITECTURE.md +8 -1
- package/docs/REV4A.md +54 -17
- package/docs/dev/API-REFERENCE.md +554 -100
- package/docs/dev/DATABASE.md +96 -0
- package/docs/dev/GATEWAY.md +21 -6
- package/docs/rag/DATA-FRESHNESS.md +6 -4
- package/docs/rag/GLOSSARY.md +12 -3
- package/docs/rag/REV4A-OVERVIEW.md +18 -5
- package/docs/rag/WHAT-I-CAN-ANSWER.md +6 -2
- package/instrumentation.ts +43 -0
- package/lib/agent-busy.ts +21 -0
- package/lib/agent-devices.ts +361 -0
- package/lib/agent-edit-state.ts +108 -0
- package/lib/agent-edit.ts +157 -0
- package/lib/agent-images.ts +375 -0
- package/lib/agent-ports-server.ts +27 -0
- package/lib/agent-ports.ts +68 -0
- package/lib/agent-recreate-state.ts +108 -0
- package/lib/agent-recreate.ts +305 -0
- package/lib/agent-restore-state.ts +107 -0
- package/lib/agent-restore.ts +135 -0
- package/lib/agent-setup.ts +66 -17
- package/lib/agent-update-state.ts +122 -0
- package/lib/agent-update.ts +448 -0
- package/lib/agent-versions.json +14 -0
- package/lib/agent-versions.ts +80 -0
- package/lib/buildAgentImage.ts +88 -290
- package/lib/channelManager.ts +149 -102
- package/lib/cold-backup.ts +354 -0
- package/lib/credentials/delivery.ts +3 -3
- package/lib/db-bootstrap.mjs +76 -0
- package/lib/docker-utils.ts +3 -3
- package/lib/provider-balance.ts +33 -12
- package/package.json +1 -1
|
@@ -0,0 +1,354 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cold backup of an agent's volume.
|
|
3
|
+
*
|
|
4
|
+
* The agent is stopped, a detached helper container archives the volume, the archive
|
|
5
|
+
* is read back, and the agent is started again if it was running. A backup taken while
|
|
6
|
+
* OpenClaw runs can copy its SQLite files mid-write; this one cannot.
|
|
7
|
+
*
|
|
8
|
+
* The helper container is the job: `rev4a-backup-<agentId>`, labelled with the agent,
|
|
9
|
+
* the archive name, the kind and whether the agent was running. Everything needed to
|
|
10
|
+
* finish or clean up a job lives in Docker, not in Rev4a's memory, so a job survives a
|
|
11
|
+
* Rev4a restart: `reconcileColdBackups()` picks helpers up again at startup, and
|
|
12
|
+
* finishing is idempotent.
|
|
13
|
+
*
|
|
14
|
+
* The archive is written as `<file>.partial` and renamed only after `tar -tzf` has read
|
|
15
|
+
* all of it back and found `.openclaw/openclaw.json`, so a listed backup is complete.
|
|
16
|
+
* Design: plans/COLD-BACKUP-PLAN.md.
|
|
17
|
+
*/
|
|
18
|
+
import { execFile } from 'child_process';
|
|
19
|
+
import { promisify } from 'util';
|
|
20
|
+
import { dockerFetch } from '@/lib/docker-socket';
|
|
21
|
+
import { isValidAgentId } from '@/lib/container';
|
|
22
|
+
import { runDocker } from '@/lib/agent-images';
|
|
23
|
+
|
|
24
|
+
const execFileAsync = promisify(execFile);
|
|
25
|
+
|
|
26
|
+
export const BACKUP_VOLUME = 'rev4a-backups';
|
|
27
|
+
const HELPER_PREFIX = 'rev4a-backup-';
|
|
28
|
+
const LABEL_AGENT = 'rev4a.backup.agent';
|
|
29
|
+
const LABEL_FILE = 'rev4a.backup.file';
|
|
30
|
+
const LABEL_KIND = 'rev4a.backup.kind';
|
|
31
|
+
const LABEL_PREV = 'rev4a.backup.prev-state';
|
|
32
|
+
const LABEL_STARTED = 'rev4a.backup.started-at';
|
|
33
|
+
/** GNU tar counts checkpoints in records of 10 240 bytes of the uncompressed stream. */
|
|
34
|
+
const TAR_RECORD_BYTES = 10_240;
|
|
35
|
+
const CHECKPOINT_RECORDS = 2_000;
|
|
36
|
+
const WATCH_MS = 3_000;
|
|
37
|
+
|
|
38
|
+
export type ColdBackupKind = 'manual' | 'preupdate' | 'prerecreate';
|
|
39
|
+
|
|
40
|
+
export interface ColdBackupState {
|
|
41
|
+
agentId: string;
|
|
42
|
+
status: 'running' | 'succeeded' | 'failed';
|
|
43
|
+
file: string;
|
|
44
|
+
kind: ColdBackupKind;
|
|
45
|
+
startedAtMs: number | null;
|
|
46
|
+
finishedAtMs: number | null;
|
|
47
|
+
/** Uncompressed bytes to archive, from `du`; null until measured. */
|
|
48
|
+
totalBytes: number | null;
|
|
49
|
+
/** Uncompressed bytes archived so far. */
|
|
50
|
+
doneBytes: number | null;
|
|
51
|
+
percent: number | null;
|
|
52
|
+
/** Archive size once complete. */
|
|
53
|
+
archiveBytes: number | null;
|
|
54
|
+
error: string | null;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export class ColdBackupBusyError extends Error {}
|
|
58
|
+
|
|
59
|
+
const helperName = (agentId: string) => `${HELPER_PREFIX}${agentId}`;
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* The helper script, run by `sh` inside the agent's own image, which carries GNU tar,
|
|
63
|
+
* coreutils and gzip. `FILE` comes in through the environment, never through the
|
|
64
|
+
* script text. Machine-readable lines start with `REV4A `.
|
|
65
|
+
*/
|
|
66
|
+
const SCRIPT = [
|
|
67
|
+
'set -u',
|
|
68
|
+
'P="/backup/$FILE.partial"',
|
|
69
|
+
'rm -f "$P"',
|
|
70
|
+
'TOTAL=$(du -sb --exclude=.npm/_cacache /source 2>/dev/null | cut -f1)',
|
|
71
|
+
'echo "REV4A total ${TOTAL:-0}"',
|
|
72
|
+
'AVAIL=$(( $(df -Pk /backup | awk \'NR==2 {print $4}\') * 1024 ))',
|
|
73
|
+
'NEED=$(( ${TOTAL:-0} * 6 / 10 ))',
|
|
74
|
+
'if [ "$AVAIL" -lt "$NEED" ]; then echo "REV4A error not enough space in rev4a-backups: about $NEED bytes needed, $AVAIL free"; exit 3; fi',
|
|
75
|
+
`if ! tar -czf "$P" --exclude=.npm/_cacache --checkpoint=${CHECKPOINT_RECORDS} --checkpoint-action=echo='REV4A progress %u' -C /source . ; then rm -f "$P"; echo "REV4A error tar failed"; exit 4; fi`,
|
|
76
|
+
'if ! tar -tzf "$P" > /tmp/rev4a-list || ! grep -qx "./.openclaw/openclaw.json" /tmp/rev4a-list; then rm -f "$P"; echo "REV4A error verification failed: the archive does not read back or holds no .openclaw/openclaw.json"; exit 5; fi',
|
|
77
|
+
'mv "$P" "/backup/$FILE" && chmod 644 "/backup/$FILE"',
|
|
78
|
+
'echo "REV4A done $(stat -c %s "/backup/$FILE")"',
|
|
79
|
+
].join('\n');
|
|
80
|
+
|
|
81
|
+
interface HelperInspect {
|
|
82
|
+
Id?: string;
|
|
83
|
+
State?: { Running?: boolean; ExitCode?: number; StartedAt?: string; FinishedAt?: string };
|
|
84
|
+
Config?: { Labels?: Record<string, string> };
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
async function inspectHelper(agentId: string): Promise<HelperInspect | null> {
|
|
88
|
+
try {
|
|
89
|
+
const info = await dockerFetch<HelperInspect>('GET', `/containers/${helperName(agentId)}/json`);
|
|
90
|
+
return typeof info?.Id === 'string' ? info : null;
|
|
91
|
+
} catch {
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Whether a cold backup of this agent is running (its helper exists). */
|
|
97
|
+
export async function isColdBackupRunning(agentId: string): Promise<boolean> {
|
|
98
|
+
const helper = await inspectHelper(agentId);
|
|
99
|
+
return !!helper?.State?.Running;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async function helperLog(agentId: string): Promise<string> {
|
|
103
|
+
try {
|
|
104
|
+
const { stdout, stderr } = await execFileAsync('docker', ['logs', '--tail', '400', helperName(agentId)], {
|
|
105
|
+
encoding: 'utf-8',
|
|
106
|
+
timeout: 15_000,
|
|
107
|
+
maxBuffer: 4 * 1024 * 1024,
|
|
108
|
+
});
|
|
109
|
+
return `${stdout}\n${stderr}`;
|
|
110
|
+
} catch {
|
|
111
|
+
return '';
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function parseLog(log: string) {
|
|
116
|
+
let totalBytes: number | null = null;
|
|
117
|
+
let doneBytes: number | null = null;
|
|
118
|
+
let archiveBytes: number | null = null;
|
|
119
|
+
let error: string | null = null;
|
|
120
|
+
for (const raw of log.split('\n')) {
|
|
121
|
+
const at = raw.indexOf('REV4A ');
|
|
122
|
+
if (at === -1) continue;
|
|
123
|
+
const [kind, ...rest] = raw.slice(at + 6).trim().split(' ');
|
|
124
|
+
const value = rest.join(' ');
|
|
125
|
+
if (kind === 'total') totalBytes = Number(value) || null;
|
|
126
|
+
else if (kind === 'progress') doneBytes = (Number(value) || 0) * TAR_RECORD_BYTES;
|
|
127
|
+
else if (kind === 'done') archiveBytes = Number(value) || null;
|
|
128
|
+
else if (kind === 'error') error = value;
|
|
129
|
+
}
|
|
130
|
+
const percent = totalBytes && doneBytes !== null ? Math.min(99, Math.round((doneBytes / totalBytes) * 100)) : null;
|
|
131
|
+
return { totalBytes, doneBytes, archiveBytes, error, percent };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function stateFrom(agentId: string, helper: HelperInspect, log: string, status: ColdBackupState['status']): ColdBackupState {
|
|
135
|
+
const labels = helper.Config?.Labels ?? {};
|
|
136
|
+
const parsed = parseLog(log);
|
|
137
|
+
const finished = helper.State?.FinishedAt && !helper.State.FinishedAt.startsWith('0001') ? Date.parse(helper.State.FinishedAt) : null;
|
|
138
|
+
return {
|
|
139
|
+
agentId,
|
|
140
|
+
status,
|
|
141
|
+
file: labels[LABEL_FILE] ?? '',
|
|
142
|
+
kind: labels[LABEL_KIND] === 'preupdate' ? 'preupdate' : labels[LABEL_KIND] === 'prerecreate' ? 'prerecreate' : 'manual',
|
|
143
|
+
startedAtMs: Number(labels[LABEL_STARTED]) || null,
|
|
144
|
+
finishedAtMs: status === 'running' ? null : finished,
|
|
145
|
+
totalBytes: parsed.totalBytes,
|
|
146
|
+
doneBytes: parsed.doneBytes,
|
|
147
|
+
percent: status === 'succeeded' ? 100 : parsed.percent,
|
|
148
|
+
archiveBytes: parsed.archiveBytes,
|
|
149
|
+
error: status === 'failed' ? parsed.error ?? `backup helper exited with code ${helper.State?.ExitCode ?? '?'}` : null,
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** Results of jobs finished since this process started, for status reads after the helper is gone. */
|
|
154
|
+
const finished = new Map<string, ColdBackupState>();
|
|
155
|
+
/** Finishing in flight, so concurrent status reads finish a job once. */
|
|
156
|
+
const finishing = new Map<string, Promise<ColdBackupState>>();
|
|
157
|
+
const watching = new Set<string>();
|
|
158
|
+
|
|
159
|
+
async function startAgentIfItWasRunning(labels: Record<string, string>, agentId: string): Promise<void> {
|
|
160
|
+
if (labels[LABEL_PREV] !== 'running') return;
|
|
161
|
+
const container = await agentContainerName(agentId);
|
|
162
|
+
if (container) await runDocker(['start', container], { timeoutMs: 60_000 }).catch(() => {});
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
async function removePartial(file: string): Promise<void> {
|
|
166
|
+
if (!file) return;
|
|
167
|
+
await runDocker(
|
|
168
|
+
['run', '--rm', '-v', `${BACKUP_VOLUME}:/backup`, '-e', `FILE=${file}`, 'alpine', 'sh', '-c', 'rm -f "/backup/$FILE.partial"'],
|
|
169
|
+
{ timeoutMs: 60_000 },
|
|
170
|
+
).catch(() => {});
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/** Finish an exited job: record its result, start the agent if it was running, remove the helper. */
|
|
174
|
+
async function finishJob(agentId: string, helper: HelperInspect): Promise<ColdBackupState> {
|
|
175
|
+
const inFlight = finishing.get(agentId);
|
|
176
|
+
if (inFlight) return inFlight;
|
|
177
|
+
const job = (async () => {
|
|
178
|
+
const log = await helperLog(agentId);
|
|
179
|
+
const ok = helper.State?.ExitCode === 0 && /REV4A done \d+/.test(log);
|
|
180
|
+
const state = stateFrom(agentId, helper, log, ok ? 'succeeded' : 'failed');
|
|
181
|
+
const labels = helper.Config?.Labels ?? {};
|
|
182
|
+
if (!ok) await removePartial(state.file);
|
|
183
|
+
await startAgentIfItWasRunning(labels, agentId);
|
|
184
|
+
await runDocker(['rm', '-f', helperName(agentId)], { timeoutMs: 30_000 }).catch(() => {});
|
|
185
|
+
finished.set(agentId, state);
|
|
186
|
+
return state;
|
|
187
|
+
})();
|
|
188
|
+
finishing.set(agentId, job);
|
|
189
|
+
try {
|
|
190
|
+
return await job;
|
|
191
|
+
} finally {
|
|
192
|
+
finishing.delete(agentId);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** Current or last cold backup of an agent; finishes the job when its helper has exited. */
|
|
197
|
+
export async function coldBackupStatus(agentId: string): Promise<ColdBackupState | null> {
|
|
198
|
+
const helper = await inspectHelper(agentId);
|
|
199
|
+
if (!helper) return finishing.get(agentId) ?? finished.get(agentId) ?? null;
|
|
200
|
+
if (helper.State?.Running) return stateFrom(agentId, helper, await helperLog(agentId), 'running');
|
|
201
|
+
return finishJob(agentId, helper);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/** Polls a running job until it finishes, so the agent restarts even when nobody reads its status. */
|
|
205
|
+
function watch(agentId: string): void {
|
|
206
|
+
if (watching.has(agentId)) return;
|
|
207
|
+
watching.add(agentId);
|
|
208
|
+
const tick = async () => {
|
|
209
|
+
const state = await coldBackupStatus(agentId).catch(() => null);
|
|
210
|
+
if (state?.status === 'running') {
|
|
211
|
+
setTimeout(() => { void tick(); }, WATCH_MS);
|
|
212
|
+
} else {
|
|
213
|
+
watching.delete(agentId);
|
|
214
|
+
}
|
|
215
|
+
};
|
|
216
|
+
setTimeout(() => { void tick(); }, WATCH_MS);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
interface ContainerInspect {
|
|
220
|
+
Name?: string;
|
|
221
|
+
Image?: string;
|
|
222
|
+
State?: { Running?: boolean };
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
async function agentContainerName(agentId: string): Promise<string | null> {
|
|
226
|
+
const filters = encodeURIComponent(JSON.stringify({ label: [`AGENT_ID=${agentId}`] }));
|
|
227
|
+
const list = await dockerFetch<{ Names?: string[] }[]>('GET', `/containers/json?all=true&filters=${filters}`).catch(() => null);
|
|
228
|
+
const name = Array.isArray(list) ? list[0]?.Names?.[0] : undefined;
|
|
229
|
+
return name ? name.replace(/^\//, '') : null;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function timestamp(): string {
|
|
233
|
+
const d = new Date();
|
|
234
|
+
const pad = (n: number, w = 2) => String(n).padStart(w, '0');
|
|
235
|
+
return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}_${pad(d.getHours())}${pad(d.getMinutes())}${pad(d.getSeconds())}${pad(d.getMilliseconds(), 3)}`;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Start a cold backup. Returns once the helper runs; the archive is written in the
|
|
240
|
+
* background. `label` goes into the file name (the OpenClaw version, for a pre-update
|
|
241
|
+
* backup): `agent-<id>-preupdate-<label>-<ts>.tar.gz` or `agent-<id>-cold-<ts>.tar.gz`.
|
|
242
|
+
* Throws `ColdBackupBusyError` when a backup of this agent is already running.
|
|
243
|
+
*/
|
|
244
|
+
export async function startColdBackup(
|
|
245
|
+
agentId: string,
|
|
246
|
+
/** `leaveStopped`: do not start the agent afterwards — the Update action recreates it next. */
|
|
247
|
+
opts: { kind: ColdBackupKind; label?: string; leaveStopped?: boolean },
|
|
248
|
+
): Promise<{ file: string }> {
|
|
249
|
+
if (!isValidAgentId(agentId)) throw new Error('Invalid agent id');
|
|
250
|
+
if (await inspectHelper(agentId)) {
|
|
251
|
+
const current = await coldBackupStatus(agentId);
|
|
252
|
+
if (current?.status === 'running') throw new ColdBackupBusyError('A backup of this agent is already running');
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
const name = await agentContainerName(agentId);
|
|
256
|
+
if (!name) throw new Error(`No container found with AGENT_ID '${agentId}'`);
|
|
257
|
+
const container = await dockerFetch<ContainerInspect>('GET', `/containers/${encodeURIComponent(name)}/json`);
|
|
258
|
+
if (!container?.Image) throw new Error(`Container '${name}' could not be inspected`);
|
|
259
|
+
const volume = `agent-${agentId}-data`;
|
|
260
|
+
const vol = await dockerFetch<{ Name?: string }>('GET', `/volumes/${volume}`).catch(() => null);
|
|
261
|
+
if (vol?.Name !== volume) throw new Error(`No persistent volume found for agent '${agentId}'`);
|
|
262
|
+
|
|
263
|
+
const label = (opts.label ?? '').replace(/[^A-Za-z0-9._-]/g, '');
|
|
264
|
+
const file = opts.kind === 'preupdate'
|
|
265
|
+
? `agent-${agentId}-preupdate-${label || 'unknown'}-${timestamp()}.tar.gz`
|
|
266
|
+
: opts.kind === 'prerecreate'
|
|
267
|
+
? `agent-${agentId}-prerecreate-${timestamp()}.tar.gz`
|
|
268
|
+
: `agent-${agentId}-cold-${timestamp()}.tar.gz`;
|
|
269
|
+
|
|
270
|
+
const wasRunning = container.State?.Running === true;
|
|
271
|
+
if (wasRunning) {
|
|
272
|
+
await runDocker(['stop', '-t', '30', name], { timeoutMs: 45_000 })
|
|
273
|
+
.catch(() => runDocker(['kill', name], { timeoutMs: 15_000 }));
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
try {
|
|
277
|
+
await runDocker([
|
|
278
|
+
'run', '-d', '--name', helperName(agentId),
|
|
279
|
+
'--label', `${LABEL_AGENT}=${agentId}`,
|
|
280
|
+
'--label', `${LABEL_FILE}=${file}`,
|
|
281
|
+
'--label', `${LABEL_KIND}=${opts.kind}`,
|
|
282
|
+
'--label', `${LABEL_PREV}=${wasRunning && !opts.leaveStopped ? 'running' : 'stopped'}`,
|
|
283
|
+
'--label', `${LABEL_STARTED}=${Date.now()}`,
|
|
284
|
+
'-e', `FILE=${file}`,
|
|
285
|
+
'-v', `${volume}:/source:ro`,
|
|
286
|
+
'-v', `${BACKUP_VOLUME}:/backup`,
|
|
287
|
+
'--entrypoint', 'sh',
|
|
288
|
+
container.Image,
|
|
289
|
+
'-c', SCRIPT,
|
|
290
|
+
], { timeoutMs: 60_000 });
|
|
291
|
+
} catch (e) {
|
|
292
|
+
await runDocker(['rm', '-f', helperName(agentId)], { timeoutMs: 30_000 }).catch(() => {});
|
|
293
|
+
if (wasRunning) await runDocker(['start', name], { timeoutMs: 60_000 }).catch(() => {});
|
|
294
|
+
throw new Error(`Could not start the backup: ${(e as Error).message}`);
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
finished.delete(agentId);
|
|
298
|
+
watch(agentId);
|
|
299
|
+
return { file };
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/** Cancel a running backup: the partial archive is removed and the agent restarted if it was running. */
|
|
303
|
+
export async function cancelColdBackup(agentId: string): Promise<boolean> {
|
|
304
|
+
const helper = await inspectHelper(agentId);
|
|
305
|
+
if (!helper?.State?.Running) return false;
|
|
306
|
+
const labels = helper.Config?.Labels ?? {};
|
|
307
|
+
await runDocker(['rm', '-f', helperName(agentId)], { timeoutMs: 30_000 }).catch(() => {});
|
|
308
|
+
await removePartial(labels[LABEL_FILE] ?? '');
|
|
309
|
+
await startAgentIfItWasRunning(labels, agentId);
|
|
310
|
+
finished.set(agentId, { ...stateFrom(agentId, helper, '', 'failed'), error: 'cancelled', finishedAtMs: Date.now() });
|
|
311
|
+
return true;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
/** Wait for a job to finish, reporting progress. Resolves with the final state. */
|
|
315
|
+
export async function waitForColdBackup(
|
|
316
|
+
agentId: string,
|
|
317
|
+
opts: { onProgress?: (state: ColdBackupState) => void; pollMs?: number; timeoutMs?: number } = {},
|
|
318
|
+
): Promise<ColdBackupState> {
|
|
319
|
+
const deadline = opts.timeoutMs ? Date.now() + opts.timeoutMs : null;
|
|
320
|
+
for (;;) {
|
|
321
|
+
const state = await coldBackupStatus(agentId);
|
|
322
|
+
if (!state) throw new Error('The backup job disappeared');
|
|
323
|
+
if (state.status !== 'running') return state;
|
|
324
|
+
opts.onProgress?.(state);
|
|
325
|
+
if (deadline && Date.now() >= deadline) {
|
|
326
|
+
// The helper is stuck: say so instead of hanging the caller (and the agent's
|
|
327
|
+
// busy lock) forever. The job is left alone — an operator can cancel it.
|
|
328
|
+
throw new Error(`The cold backup did not finish within ${Math.round(opts.timeoutMs! / 60_000)} minutes`);
|
|
329
|
+
}
|
|
330
|
+
await new Promise((r) => setTimeout(r, opts.pollMs ?? 2_000));
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
/** Agent ids with a backup helper, running or exited. */
|
|
335
|
+
export async function listColdBackupHelpers(): Promise<string[]> {
|
|
336
|
+
const filters = encodeURIComponent(JSON.stringify({ label: [LABEL_AGENT] }));
|
|
337
|
+
const list = await dockerFetch<{ Labels?: Record<string, string> }[]>('GET', `/containers/json?all=true&filters=${filters}`).catch(() => null);
|
|
338
|
+
return (Array.isArray(list) ? list : []).map((c) => c.Labels?.[LABEL_AGENT] ?? '').filter(Boolean);
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/** Agent ids with a cold backup currently running (helper container still running). */
|
|
342
|
+
export async function listRunningColdBackups(): Promise<string[]> {
|
|
343
|
+
const filters = encodeURIComponent(JSON.stringify({ label: [LABEL_AGENT], status: ['running'] }));
|
|
344
|
+
const list = await dockerFetch<{ Labels?: Record<string, string> }[]>('GET', `/containers/json?all=true&filters=${filters}`).catch(() => null);
|
|
345
|
+
return (Array.isArray(list) ? list : []).map((c) => c.Labels?.[LABEL_AGENT] ?? '').filter(Boolean);
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
/** At startup: finish helpers that exited while Rev4a was down, and watch the ones still running. */
|
|
349
|
+
export async function reconcileColdBackups(): Promise<void> {
|
|
350
|
+
for (const agentId of await listColdBackupHelpers()) {
|
|
351
|
+
const state = await coldBackupStatus(agentId).catch(() => null);
|
|
352
|
+
if (state?.status === 'running') watch(agentId);
|
|
353
|
+
}
|
|
354
|
+
}
|
|
@@ -149,7 +149,7 @@ async function deliverGithub(container: string, secret: Record<string, string>):
|
|
|
149
149
|
if (!token) throw new Error('Missing token');
|
|
150
150
|
|
|
151
151
|
// Require gh CLI preinstalled in the agent image
|
|
152
|
-
if (!await cliPresent(container, 'gh')) throw new Error('gh CLI not installed in this container.
|
|
152
|
+
if (!await cliPresent(container, 'gh')) throw new Error('gh CLI not installed in this container. Download the current agent image from the Agents page and update this agent.');
|
|
153
153
|
|
|
154
154
|
await containerExec(
|
|
155
155
|
container,
|
|
@@ -162,7 +162,7 @@ async function deliverVercel(container: string, secret: Record<string, string>):
|
|
|
162
162
|
if (!token) throw new Error('Missing token');
|
|
163
163
|
|
|
164
164
|
// Require vercel CLI preinstalled in the agent image
|
|
165
|
-
if (!await cliPresent(container, 'vercel')) throw new Error('vercel CLI not installed in this container.
|
|
165
|
+
if (!await cliPresent(container, 'vercel')) throw new Error('vercel CLI not installed in this container. Download the current agent image from the Agents page and update this agent.');
|
|
166
166
|
|
|
167
167
|
// Validate token and discover team scope via Node fetch (no curl/exec)
|
|
168
168
|
const scope = await discoverVercelScope(token);
|
|
@@ -191,7 +191,7 @@ async function deliverSupabase(container: string, secret: Record<string, string>
|
|
|
191
191
|
if (!accessToken || !accessToken.trim()) throw new Error('Missing Supabase access token');
|
|
192
192
|
|
|
193
193
|
// Check supabase CLI is available (installed globally in the image)
|
|
194
|
-
if (!await cliPresent(container, 'supabase')) throw new Error('supabase CLI not installed in this container.
|
|
194
|
+
if (!await cliPresent(container, 'supabase')) throw new Error('supabase CLI not installed in this container. Download the current agent image from the Agents page and update this agent.');
|
|
195
195
|
|
|
196
196
|
// Login with access token — grants access to all projects on the account
|
|
197
197
|
// supabase login --token supports non-interactive auth
|
package/lib/db-bootstrap.mjs
CHANGED
|
@@ -139,6 +139,82 @@ export function initializeRev4aDb(dbPath) {
|
|
|
139
139
|
['model', 'TEXT'],
|
|
140
140
|
]);
|
|
141
141
|
|
|
142
|
+
// Agent updates: one row per attempt to move an agent to a newer OpenClaw version
|
|
143
|
+
// (lib/agent-update.ts). It answers, after any restart, where an update stopped and
|
|
144
|
+
// which archive restores the agent.
|
|
145
|
+
db.exec(`
|
|
146
|
+
CREATE TABLE IF NOT EXISTS agent_upgrades (
|
|
147
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
148
|
+
agent_id TEXT NOT NULL,
|
|
149
|
+
status TEXT NOT NULL,
|
|
150
|
+
from_version TEXT,
|
|
151
|
+
to_version TEXT NOT NULL,
|
|
152
|
+
backup_file TEXT,
|
|
153
|
+
baseline_json TEXT,
|
|
154
|
+
verify_json TEXT,
|
|
155
|
+
error TEXT,
|
|
156
|
+
started_at INTEGER NOT NULL,
|
|
157
|
+
updated_at INTEGER NOT NULL,
|
|
158
|
+
finished_at INTEGER
|
|
159
|
+
);
|
|
160
|
+
CREATE INDEX IF NOT EXISTS idx_agent_upgrades_agent ON agent_upgrades(agent_id, started_at);
|
|
161
|
+
`);
|
|
162
|
+
|
|
163
|
+
// Recreate jobs (lib/agent-recreate.ts): same shape as an update minus the
|
|
164
|
+
// migration — a cold backup, the container rebuilt on the same version, the
|
|
165
|
+
// gateway started again. Persisted so a reload or a Rev4a restart mid-way is
|
|
166
|
+
// visible instead of lost, and so an interrupted job is marked at startup.
|
|
167
|
+
db.exec(`
|
|
168
|
+
CREATE TABLE IF NOT EXISTS agent_recreates (
|
|
169
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
170
|
+
agent_id TEXT NOT NULL,
|
|
171
|
+
status TEXT NOT NULL,
|
|
172
|
+
image TEXT,
|
|
173
|
+
backup_file TEXT,
|
|
174
|
+
error TEXT,
|
|
175
|
+
started_at INTEGER NOT NULL,
|
|
176
|
+
updated_at INTEGER NOT NULL,
|
|
177
|
+
finished_at INTEGER
|
|
178
|
+
);
|
|
179
|
+
CREATE INDEX IF NOT EXISTS idx_agent_recreates_agent ON agent_recreates(agent_id, started_at);
|
|
180
|
+
`);
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
// Restore jobs (lib/agent-restore.ts): the volume is replaced from an archive, so the
|
|
184
|
+
// job is visible and survives a reload or a Rev4a restart like a recreate does.
|
|
185
|
+
db.exec(`
|
|
186
|
+
CREATE TABLE IF NOT EXISTS agent_restores (
|
|
187
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
188
|
+
agent_id TEXT NOT NULL,
|
|
189
|
+
status TEXT NOT NULL,
|
|
190
|
+
file TEXT,
|
|
191
|
+
error TEXT,
|
|
192
|
+
started_at INTEGER NOT NULL,
|
|
193
|
+
updated_at INTEGER NOT NULL,
|
|
194
|
+
finished_at INTEGER
|
|
195
|
+
);
|
|
196
|
+
CREATE INDEX IF NOT EXISTS idx_agent_restores_agent ON agent_restores(agent_id, started_at);
|
|
197
|
+
`);
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
// Edit jobs (lib/agent-edit.ts): the container is rebuilt with a new display name
|
|
201
|
+
// and/or port range, no volume backup involved. Persisted like the others so a
|
|
202
|
+
// reload or a Rev4a restart never loses track of it.
|
|
203
|
+
db.exec(`
|
|
204
|
+
CREATE TABLE IF NOT EXISTS agent_edits (
|
|
205
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
206
|
+
agent_id TEXT NOT NULL,
|
|
207
|
+
status TEXT NOT NULL,
|
|
208
|
+
display_name TEXT,
|
|
209
|
+
port_range TEXT,
|
|
210
|
+
error TEXT,
|
|
211
|
+
started_at INTEGER NOT NULL,
|
|
212
|
+
updated_at INTEGER NOT NULL,
|
|
213
|
+
finished_at INTEGER
|
|
214
|
+
);
|
|
215
|
+
CREATE INDEX IF NOT EXISTS idx_agent_edits_agent ON agent_edits(agent_id, started_at);
|
|
216
|
+
`);
|
|
217
|
+
|
|
142
218
|
|
|
143
219
|
db.close();
|
|
144
220
|
}
|
package/lib/docker-utils.ts
CHANGED
|
@@ -19,9 +19,9 @@
|
|
|
19
19
|
* user's browser open", not "what is this machine's WAN address". Those
|
|
20
20
|
* differ behind NAT, behind a reverse proxy, and over a tunnel.
|
|
21
21
|
*
|
|
22
|
-
* Agent
|
|
23
|
-
*
|
|
24
|
-
* server, since the dashboard is being served over it.
|
|
22
|
+
* Agent Control UI links are now built from the host the request reached Rev4a on
|
|
23
|
+
* (`hostnameFromHostHeader()` in `lib/agent-devices.ts`) — a hostname already
|
|
24
|
+
* proven to reach this server, since the dashboard is being served over it.
|
|
25
25
|
*/
|
|
26
26
|
|
|
27
27
|
/**
|
package/lib/provider-balance.ts
CHANGED
|
@@ -27,6 +27,9 @@ interface BalanceInfo {
|
|
|
27
27
|
granted_balance?: string;
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
+
/** Thrown when the provider has no balance API for this account type (permanent, not transient). */
|
|
31
|
+
export class BalanceUnsupportedError extends Error {}
|
|
32
|
+
|
|
30
33
|
/* ------------------------------------------------------------------ */
|
|
31
34
|
/* DeepSeek */
|
|
32
35
|
/* ------------------------------------------------------------------ */
|
|
@@ -99,7 +102,21 @@ async function fetchZAIBalance(apiKey: string): Promise<BalanceResult> {
|
|
|
99
102
|
if (!resp.ok) {
|
|
100
103
|
throw new Error(`Z.AI quota API returned ${resp.status}`);
|
|
101
104
|
}
|
|
102
|
-
const json = await resp.json() as {
|
|
105
|
+
const json = await resp.json() as {
|
|
106
|
+
success?: boolean;
|
|
107
|
+
msg?: string;
|
|
108
|
+
limits?: ZAILimit[];
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
// The endpoint only serves GLM Coding Plan subscriptions. A pay-as-you-go
|
|
112
|
+
// account gets HTTP 200 with success:false and no limits; Z.AI exposes no
|
|
113
|
+
// balance API for it (no undocumented endpoint either, verified 2026-09-16),
|
|
114
|
+
// so this is permanent, not transient: the UI hides the badge entirely.
|
|
115
|
+
if (json.success === false) {
|
|
116
|
+
throw new BalanceUnsupportedError(
|
|
117
|
+
`Z.AI exposes no balance API for pay-as-you-go accounts (the token-quota endpoint requires a Coding Plan)`,
|
|
118
|
+
);
|
|
119
|
+
}
|
|
103
120
|
|
|
104
121
|
const tokenLimit = json.limits?.find((l) => l.limitType === 'TOKENS_LIMIT');
|
|
105
122
|
if (tokenLimit) {
|
|
@@ -116,17 +133,21 @@ async function fetchZAIBalance(apiKey: string): Promise<BalanceResult> {
|
|
|
116
133
|
};
|
|
117
134
|
}
|
|
118
135
|
|
|
119
|
-
// Fallback: sum all limits
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
136
|
+
// Fallback: sum all limits, only when the API actually returned some.
|
|
137
|
+
if ((json.limits ?? []).length > 0) {
|
|
138
|
+
const totalRemaining = (json.limits ?? []).reduce(
|
|
139
|
+
(sum, l) => sum + (l.remaining ?? l.limitValue - l.usage),
|
|
140
|
+
0,
|
|
141
|
+
);
|
|
142
|
+
return {
|
|
143
|
+
balance: `${(totalRemaining / 1_000_000).toFixed(1)}M tokens`,
|
|
144
|
+
currency: 'TOKENS',
|
|
145
|
+
value: totalRemaining,
|
|
146
|
+
label: 'Remaining',
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
throw new Error('Z.AI returned no quota limits');
|
|
130
151
|
}
|
|
131
152
|
|
|
132
153
|
/* ------------------------------------------------------------------ */
|