@flame0510/project-aether 1.9.1 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/app/agents/CostSection.tsx +100 -0
- package/app/agents/PageClient.tsx +7 -0
- package/app/api/assistant/route.ts +8 -1
- package/app/api/costs/agent/route.ts +56 -0
- package/app/api/costs/route.ts +19 -77
- package/app/api/costs/usage/route.ts +65 -0
- package/app/api/gateway/sync.ts +18 -4
- package/app/api/stats/route.ts +2 -3
- package/app/api/stats-since/route.ts +1 -1
- package/app/api/stream/route.ts +3 -5
- package/app/api/system-health/route.ts +33 -32
- package/app/components/CostBreakdown.tsx +1 -1
- package/app/components/Sidebar.tsx +10 -0
- package/app/components/SystemCockpit.tsx +1 -1
- package/app/components/ui/TimeSeriesChart.tsx +27 -9
- package/app/costs/CostsSkeleton.tsx +88 -0
- package/app/costs/PageClient.tsx +366 -0
- package/app/costs/loading.tsx +13 -0
- package/app/costs/page.tsx +5 -0
- package/app/globals.css +6 -0
- package/daemon.js +455 -41
- package/docs/ARCHITECTURE.md +39 -29
- package/docs/DESIGN-SYSTEM.md +2 -2
- package/docs/FRONTEND-ARCHITECTURE.md +4 -2
- package/docs/REV4A.md +3 -2
- package/docs/dev/API-REFERENCE.md +118 -22
- package/docs/dev/DATABASE.md +119 -19
- package/docs/dev/GATEWAY.md +53 -9
- package/docs/rag/DATA-FRESHNESS.md +19 -7
- package/docs/rag/GLOSSARY.md +7 -4
- package/docs/rag/REV4A-OVERVIEW.md +4 -1
- package/docs/rag/WHAT-I-CAN-ANSWER.md +3 -3
- package/instrumentation.ts +11 -0
- package/lib/agent-costs.ts +172 -0
- package/lib/cost-reconciliation.ts +78 -0
- package/lib/costs-db.ts +31 -0
- package/lib/model-pricing.ts +123 -3
- package/lib/price-schedule-sync.ts +133 -0
- package/lib/utils/format.ts +12 -0
- package/model-pricing.json +269 -121
- package/package.json +1 -1
- package/scripts/refresh-model-pricing.mjs +16 -4
- package/lib/billing.ts +0 -100
package/daemon.js
CHANGED
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
/**
|
|
3
3
|
* Rev4a Daemon — SQLite event logger
|
|
4
|
-
* Polls OpenClaw sessions every 30s, writes events to SQLite,
|
|
5
|
-
*
|
|
4
|
+
* Polls OpenClaw sessions every 30s, writes events to SQLite, samples the host machine
|
|
5
|
+
* (CPU, RAM, swap, storage) every 30s, and every 5 min reads what each running agent
|
|
6
|
+
* spent, as its OpenClaw priced it, into costs.db — each on a timer of its own.
|
|
6
7
|
*/
|
|
7
8
|
|
|
8
9
|
'use strict';
|
|
9
10
|
|
|
10
11
|
const os = require('os');
|
|
11
12
|
const fs = require('fs');
|
|
12
|
-
const { spawnSync, execSync } = require('child_process');
|
|
13
|
+
const { spawnSync, execSync, execFile } = require('child_process');
|
|
13
14
|
const Database = require('better-sqlite3');
|
|
14
15
|
const path = require('path');
|
|
15
16
|
|
|
@@ -22,41 +23,8 @@ const DATA_DIR = process.env.REV4A_DATA_DIR
|
|
|
22
23
|
const DB_PATH = process.env.REV4A_DB || path.join(DATA_DIR, 'data', 'events.db');
|
|
23
24
|
const POLL_INTERVAL_MS = 30_000;
|
|
24
25
|
|
|
25
|
-
//
|
|
26
|
-
|
|
27
|
-
'claude-sonnet-4': { in: 3.00, out: 15.00 },
|
|
28
|
-
'claude-opus-4': { in: 15.00, out: 75.00 },
|
|
29
|
-
'gpt-5-mini': { in: 0.15, out: 0.60 },
|
|
30
|
-
'codex': { in: 3.00, out: 15.00 },
|
|
31
|
-
'gemini': { in: 0.075, out: 0.30 },
|
|
32
|
-
'flash': { in: 0.075, out: 0.30 },
|
|
33
|
-
'deepseek': { in: 0.55, out: 2.19 },
|
|
34
|
-
'default': { in: 3.00, out: 15.00 },
|
|
35
|
-
};
|
|
36
|
-
|
|
37
|
-
// Direct aliases for models that do not match by substring
|
|
38
|
-
const MODEL_ALIASES = {
|
|
39
|
-
'cheap': 'flash',
|
|
40
|
-
'fast': 'claude-sonnet-4',
|
|
41
|
-
'big': 'claude-opus-4',
|
|
42
|
-
'coder': 'codex',
|
|
43
|
-
'pro': 'gemini',
|
|
44
|
-
'reason': 'deepseek',
|
|
45
|
-
};
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
function estimateCost(model, tokensIn, tokensOut) {
|
|
49
|
-
if (!tokensIn && !tokensOut) return 0;
|
|
50
|
-
const m = (model || '').toLowerCase();
|
|
51
|
-
// Check direct aliases first (e.g. 'cheap' → 'flash')
|
|
52
|
-
const aliasKey = Object.keys(MODEL_ALIASES).find(a => m === a || m.endsWith('/' + a));
|
|
53
|
-
const resolved = aliasKey ? MODEL_ALIASES[aliasKey] : null;
|
|
54
|
-
const key = resolved
|
|
55
|
-
|| Object.keys(MODEL_PRICING).find(k => k !== 'default' && m.includes(k))
|
|
56
|
-
|| 'default';
|
|
57
|
-
const p = MODEL_PRICING[key];
|
|
58
|
-
return ((tokensIn || 0) / 1_000_000 * p.in) + ((tokensOut || 0) / 1_000_000 * p.out);
|
|
59
|
-
}
|
|
26
|
+
// No cost is estimated here. What agents spend is priced by each agent's own OpenClaw
|
|
27
|
+
// and read by the Agent Costs timer below into costs.db; sessions.cost_usd stays 0.
|
|
60
28
|
|
|
61
29
|
// ── DB Setup ────────────────────────────────────────────────────────────────
|
|
62
30
|
|
|
@@ -448,6 +416,435 @@ function collectSystemMetrics() {
|
|
|
448
416
|
}
|
|
449
417
|
}
|
|
450
418
|
|
|
419
|
+
// ── Agent Costs ──────────────────────────────────────────────────────────────
|
|
420
|
+
//
|
|
421
|
+
// What each agent spent, as its own OpenClaw computed it. Rev4a writes every model's
|
|
422
|
+
// price into the synced `rev4a` provider block (app/api/gateway/sync.ts); OpenClaw then
|
|
423
|
+
// prices each call, session and day itself. This timer reads those figures from each
|
|
424
|
+
// running agent's Gateway and keeps them, so a stopped or deleted agent keeps its
|
|
425
|
+
// history. Nothing here computes a cost.
|
|
426
|
+
|
|
427
|
+
/**
|
|
428
|
+
* Its own database next to events.db, like metrics.db: the daemon is the only writer;
|
|
429
|
+
* the API reads it through lib/costs-db.ts (same path) — the Costs page, the dashboard,
|
|
430
|
+
* system-health and the agent panel. Days are UTC dates, as
|
|
431
|
+
* OpenClaw buckets them by default and as DeepSeek schedules its rates.
|
|
432
|
+
*
|
|
433
|
+
* Opened in a try: a costs.db that cannot be opened turns collection off with a log line.
|
|
434
|
+
*/
|
|
435
|
+
const COSTS_DB_PATH = path.join(path.dirname(DB_PATH), 'costs.db');
|
|
436
|
+
const costs = openCostsStore();
|
|
437
|
+
|
|
438
|
+
function openCostsStore() {
|
|
439
|
+
try {
|
|
440
|
+
const cdb = new Database(COSTS_DB_PATH);
|
|
441
|
+
cdb.pragma('journal_mode = WAL');
|
|
442
|
+
cdb.pragma('synchronous = NORMAL');
|
|
443
|
+
cdb.exec(`
|
|
444
|
+
CREATE TABLE IF NOT EXISTS agent_cost_daily (
|
|
445
|
+
container TEXT NOT NULL,
|
|
446
|
+
date TEXT NOT NULL,
|
|
447
|
+
input INTEGER NOT NULL DEFAULT 0,
|
|
448
|
+
output INTEGER NOT NULL DEFAULT 0,
|
|
449
|
+
cache_read INTEGER NOT NULL DEFAULT 0,
|
|
450
|
+
cache_write INTEGER NOT NULL DEFAULT 0,
|
|
451
|
+
total_tokens INTEGER NOT NULL DEFAULT 0,
|
|
452
|
+
cost REAL NOT NULL DEFAULT 0,
|
|
453
|
+
input_cost REAL NOT NULL DEFAULT 0,
|
|
454
|
+
output_cost REAL NOT NULL DEFAULT 0,
|
|
455
|
+
cache_read_cost REAL NOT NULL DEFAULT 0,
|
|
456
|
+
cache_write_cost REAL NOT NULL DEFAULT 0,
|
|
457
|
+
missing INTEGER NOT NULL DEFAULT 0,
|
|
458
|
+
PRIMARY KEY (container, date)
|
|
459
|
+
);
|
|
460
|
+
CREATE TABLE IF NOT EXISTS agent_cost_model_daily (
|
|
461
|
+
container TEXT NOT NULL,
|
|
462
|
+
date TEXT NOT NULL,
|
|
463
|
+
provider TEXT NOT NULL,
|
|
464
|
+
model TEXT NOT NULL,
|
|
465
|
+
tokens INTEGER NOT NULL DEFAULT 0,
|
|
466
|
+
cost REAL NOT NULL DEFAULT 0,
|
|
467
|
+
calls INTEGER NOT NULL DEFAULT 0,
|
|
468
|
+
PRIMARY KEY (container, date, provider, model)
|
|
469
|
+
);
|
|
470
|
+
CREATE TABLE IF NOT EXISTS agent_cost_sessions (
|
|
471
|
+
container TEXT NOT NULL,
|
|
472
|
+
session_id TEXT NOT NULL,
|
|
473
|
+
session_key TEXT,
|
|
474
|
+
label TEXT,
|
|
475
|
+
agent_id TEXT,
|
|
476
|
+
provider TEXT,
|
|
477
|
+
model TEXT,
|
|
478
|
+
tokens INTEGER NOT NULL DEFAULT 0,
|
|
479
|
+
cost REAL NOT NULL DEFAULT 0,
|
|
480
|
+
missing INTEGER NOT NULL DEFAULT 0,
|
|
481
|
+
first_activity INTEGER,
|
|
482
|
+
last_activity INTEGER,
|
|
483
|
+
PRIMARY KEY (container, session_id)
|
|
484
|
+
);
|
|
485
|
+
CREATE INDEX IF NOT EXISTS idx_cost_sessions_last ON agent_cost_sessions(last_activity);
|
|
486
|
+
-- The vendor's own figure next to what the agents priced, taken together (see
|
|
487
|
+
-- collectVendorBalances): the difference between two rows compares the same interval.
|
|
488
|
+
CREATE TABLE IF NOT EXISTS vendor_balance (
|
|
489
|
+
ts INTEGER NOT NULL,
|
|
490
|
+
vendor TEXT NOT NULL,
|
|
491
|
+
currency TEXT,
|
|
492
|
+
balance REAL,
|
|
493
|
+
usage REAL,
|
|
494
|
+
agents_cost REAL NOT NULL DEFAULT 0,
|
|
495
|
+
PRIMARY KEY (vendor, ts)
|
|
496
|
+
);
|
|
497
|
+
CREATE TABLE IF NOT EXISTS agent_cost_sources (
|
|
498
|
+
container TEXT PRIMARY KEY,
|
|
499
|
+
agent_id TEXT,
|
|
500
|
+
name TEXT,
|
|
501
|
+
collected_at INTEGER,
|
|
502
|
+
attempted_at INTEGER,
|
|
503
|
+
error TEXT,
|
|
504
|
+
missing_by_model TEXT
|
|
505
|
+
);
|
|
506
|
+
`);
|
|
507
|
+
return {
|
|
508
|
+
db: cdb,
|
|
509
|
+
clearDaily: cdb.prepare('DELETE FROM agent_cost_daily WHERE container = ? AND date >= ?'),
|
|
510
|
+
clearModelDaily: cdb.prepare('DELETE FROM agent_cost_model_daily WHERE container = ? AND date >= ?'),
|
|
511
|
+
insertDaily: cdb.prepare(`
|
|
512
|
+
INSERT INTO agent_cost_daily (container, date, input, output, cache_read, cache_write, total_tokens,
|
|
513
|
+
cost, input_cost, output_cost, cache_read_cost, cache_write_cost, missing)
|
|
514
|
+
VALUES (@container, @date, @input, @output, @cache_read, @cache_write, @total_tokens,
|
|
515
|
+
@cost, @input_cost, @output_cost, @cache_read_cost, @cache_write_cost, @missing)
|
|
516
|
+
`),
|
|
517
|
+
insertModelDaily: cdb.prepare(`
|
|
518
|
+
INSERT OR REPLACE INTO agent_cost_model_daily (container, date, provider, model, tokens, cost, calls)
|
|
519
|
+
VALUES (@container, @date, @provider, @model, @tokens, @cost, @calls)
|
|
520
|
+
`),
|
|
521
|
+
upsertSession: cdb.prepare(`
|
|
522
|
+
INSERT INTO agent_cost_sessions (container, session_id, session_key, label, agent_id, provider, model,
|
|
523
|
+
tokens, cost, missing, first_activity, last_activity)
|
|
524
|
+
VALUES (@container, @session_id, @session_key, @label, @agent_id, @provider, @model,
|
|
525
|
+
@tokens, @cost, @missing, @first_activity, @last_activity)
|
|
526
|
+
ON CONFLICT(container, session_id) DO UPDATE SET
|
|
527
|
+
session_key = excluded.session_key, label = excluded.label, agent_id = excluded.agent_id,
|
|
528
|
+
provider = excluded.provider, model = excluded.model, tokens = excluded.tokens, cost = excluded.cost,
|
|
529
|
+
missing = excluded.missing, first_activity = excluded.first_activity, last_activity = excluded.last_activity
|
|
530
|
+
`),
|
|
531
|
+
insertBalance: cdb.prepare(`
|
|
532
|
+
INSERT OR REPLACE INTO vendor_balance (ts, vendor, currency, balance, usage, agents_cost)
|
|
533
|
+
VALUES (@ts, @vendor, @currency, @balance, @usage, @agents_cost)
|
|
534
|
+
`),
|
|
535
|
+
lastBalanceTs: cdb.prepare('SELECT MAX(ts) AS ts FROM vendor_balance WHERE vendor = ?'),
|
|
536
|
+
pruneBalances: cdb.prepare('DELETE FROM vendor_balance WHERE ts < ?'),
|
|
537
|
+
agentsCostFor: cdb.prepare(`SELECT COALESCE(SUM(cost), 0) AS cost FROM agent_cost_model_daily WHERE model LIKE ? || '/%'`),
|
|
538
|
+
upsertSource: cdb.prepare(`
|
|
539
|
+
INSERT INTO agent_cost_sources (container, agent_id, name, collected_at, attempted_at, error, missing_by_model)
|
|
540
|
+
VALUES (@container, @agent_id, @name, @collected_at, @attempted_at, @error, @missing_by_model)
|
|
541
|
+
ON CONFLICT(container) DO UPDATE SET
|
|
542
|
+
agent_id = excluded.agent_id, name = excluded.name,
|
|
543
|
+
collected_at = COALESCE(excluded.collected_at, agent_cost_sources.collected_at),
|
|
544
|
+
attempted_at = excluded.attempted_at, error = excluded.error,
|
|
545
|
+
missing_by_model = COALESCE(excluded.missing_by_model, agent_cost_sources.missing_by_model)
|
|
546
|
+
`),
|
|
547
|
+
};
|
|
548
|
+
} catch (err) {
|
|
549
|
+
log(`[COSTS] disabled: cannot open ${COSTS_DB_PATH}: ${err.message}`);
|
|
550
|
+
return null;
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
const COSTS_INTERVAL_MS = 5 * 60 * 1000;
|
|
555
|
+
const COSTS_FIRST_DELAY_MS = 20_000;
|
|
556
|
+
/** Days re-read on every pass: late calls and repriced ones land in the right day. */
|
|
557
|
+
const COSTS_WINDOW_DAYS = 30;
|
|
558
|
+
/** The most recent sessions kept per pass; older ones stay as last read. */
|
|
559
|
+
const COSTS_SESSIONS_LIMIT = 50;
|
|
560
|
+
/**
|
|
561
|
+
* OpenClaw answers from an index of its transcripts, rebuilt in the background after a
|
|
562
|
+
* price change or a restart; until then it answers zeros with `cacheStatus` not `fresh`.
|
|
563
|
+
* Such an answer is never stored: ask again a little later, a few times. Measured on a
|
|
564
|
+
* VPS (2026-09-27): about 60 s to rebuild, from 4 to 290 transcript files alike.
|
|
565
|
+
*/
|
|
566
|
+
const COSTS_FRESH_ATTEMPTS = 6;
|
|
567
|
+
const COSTS_FRESH_WAIT_MS = 20_000;
|
|
568
|
+
const COSTS_CALL_TIMEOUT_MS = 60_000;
|
|
569
|
+
|
|
570
|
+
/**
|
|
571
|
+
* Runs `openclaw "$@"` against the agent's own Gateway on port 3000, with the token
|
|
572
|
+
* resolved inside the container as the entrypoint does (lib/agent-devices.ts, same
|
|
573
|
+
* script): it never reaches argv on the host. Arguments stay positional.
|
|
574
|
+
*/
|
|
575
|
+
const COSTS_GATEWAY_CLI = [
|
|
576
|
+
'export OPENCLAW_GATEWAY_PORT=3000',
|
|
577
|
+
'if [ -f /root/.agent-token ]; then OPENCLAW_GATEWAY_TOKEN=$(head -1 /root/.agent-token | tr -d "[:space:]"); export OPENCLAW_GATEWAY_TOKEN; fi',
|
|
578
|
+
'exec openclaw "$@"',
|
|
579
|
+
].join('; ');
|
|
580
|
+
|
|
581
|
+
const EXEC_MAX_BUFFER = 32 * 1024 * 1024;
|
|
582
|
+
|
|
583
|
+
/**
|
|
584
|
+
* Why a command failed, in words a person can act on. `openclaw gateway call --json`
|
|
585
|
+
* prints its error as JSON on stdout (`{"ok":false,"error":{"message":…}}`) and exits 1,
|
|
586
|
+
* with stderr often empty; Node's own message is the whole command line. So: the JSON
|
|
587
|
+
* error, else the last stderr line, else the kind of failure — never the argv.
|
|
588
|
+
*/
|
|
589
|
+
function failureReason(err, stdout, stderr) {
|
|
590
|
+
if (err.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER') return 'answer larger than the read limit';
|
|
591
|
+
if (err.killed) return 'timed out';
|
|
592
|
+
const out = String(stdout || '');
|
|
593
|
+
const start = out.indexOf('{');
|
|
594
|
+
if (start >= 0) {
|
|
595
|
+
try {
|
|
596
|
+
const j = JSON.parse(out.slice(start));
|
|
597
|
+
const m = j?.error?.message ?? (typeof j?.error === 'string' ? j.error : null);
|
|
598
|
+
if (m) return String(m).split('\n')[0];
|
|
599
|
+
} catch { /* not JSON: fall through */ }
|
|
600
|
+
}
|
|
601
|
+
const line = String(stderr || '').trim().split('\n').filter(Boolean).pop();
|
|
602
|
+
return line || `exited with code ${err.code ?? 'unknown'}`;
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
function execFileAsync(cmd, args, timeoutMs) {
|
|
606
|
+
return new Promise((resolve, reject) => {
|
|
607
|
+
execFile(cmd, args, { timeout: timeoutMs, maxBuffer: EXEC_MAX_BUFFER, encoding: 'utf-8' }, (err, stdout, stderr) => {
|
|
608
|
+
if (err) return reject(new Error(failureReason(err, stdout, stderr)));
|
|
609
|
+
resolve(stdout);
|
|
610
|
+
});
|
|
611
|
+
});
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
async function gatewayCall(container, method, params) {
|
|
615
|
+
const out = await execFileAsync('docker', ['exec', container, 'sh', '-c', COSTS_GATEWAY_CLI, 'sh',
|
|
616
|
+
'gateway', 'call', method, '--json', '--timeout', String(COSTS_CALL_TIMEOUT_MS - 5000), '--params', JSON.stringify(params)],
|
|
617
|
+
COSTS_CALL_TIMEOUT_MS);
|
|
618
|
+
const start = out.indexOf('{');
|
|
619
|
+
if (start < 0) throw new Error(`${method}: no JSON in the answer`);
|
|
620
|
+
return JSON.parse(out.slice(start));
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
/** A Docker container name, as `docker ps` prints it; anything else is never exec'd. */
|
|
624
|
+
const CONTAINER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/;
|
|
625
|
+
/**
|
|
626
|
+
* Each field quoted and escaped by Docker itself (`json`, `printf "%q"`), so a display
|
|
627
|
+
* name holding a tab or a newline stays one field of one line. Only AGENT_NAME is read
|
|
628
|
+
* from the environment.
|
|
629
|
+
*/
|
|
630
|
+
const INSPECT_FORMAT = '{{json .Name}}\t{{json (index .Config.Labels "AGENT_ID")}}\t'
|
|
631
|
+
+ '{{range .Config.Env}}{{if eq (index (split . "=") 0) "AGENT_NAME"}}{{printf "%q" .}}{{end}}{{end}}';
|
|
632
|
+
|
|
633
|
+
function parseQuoted(field) {
|
|
634
|
+
if (!field) return null;
|
|
635
|
+
try { return JSON.parse(field); } catch { return null; }
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
/** Running agent containers with their AGENT_ID and AGENT_NAME. */
|
|
639
|
+
async function listRunningAgents() {
|
|
640
|
+
const names = (await execFileAsync('docker', ['ps', '--filter', 'label=AGENT_ID', '--format', '{{.Names}}'], 10_000))
|
|
641
|
+
.split('\n').map((s) => s.trim()).filter((n) => CONTAINER_NAME_RE.test(n));
|
|
642
|
+
if (!names.length) return [];
|
|
643
|
+
let raw;
|
|
644
|
+
try {
|
|
645
|
+
raw = await execFileAsync('docker', ['inspect', '--format', INSPECT_FORMAT, ...names], 10_000);
|
|
646
|
+
} catch {
|
|
647
|
+
// One container removed between `ps` and `inspect` fails the batch: ask one by one.
|
|
648
|
+
const parts = [];
|
|
649
|
+
for (const n of names) {
|
|
650
|
+
parts.push(await execFileAsync('docker', ['inspect', '--format', INSPECT_FORMAT, n], 10_000).catch(() => ''));
|
|
651
|
+
}
|
|
652
|
+
raw = parts.join('\n');
|
|
653
|
+
}
|
|
654
|
+
return raw.split('\n').filter(Boolean).map((line) => {
|
|
655
|
+
const [name, agentId, env] = line.split('\t');
|
|
656
|
+
const container = String(parseQuoted(name) ?? '').replace(/^\//, '');
|
|
657
|
+
const agentName = String(parseQuoted(env) ?? '').replace(/^AGENT_NAME=/, '').replace(/[\u0000-\u001f\u007f]/g, ' ').trim();
|
|
658
|
+
return { container, agentId: parseQuoted(agentId) || null, name: agentName || null };
|
|
659
|
+
}).filter((a) => CONTAINER_NAME_RE.test(a.container));
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
const utcDate = (ms) => new Date(ms).toISOString().slice(0, 10);
|
|
663
|
+
const num = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : 0);
|
|
664
|
+
|
|
665
|
+
/**
|
|
666
|
+
* An answer to store: its index is `fresh`, or there is no index to speak of — an agent
|
|
667
|
+
* with nothing in the range answers without `cacheStatus` at all.
|
|
668
|
+
*/
|
|
669
|
+
function isFresh(answer) {
|
|
670
|
+
const status = answer?.cacheStatus?.status;
|
|
671
|
+
return status === undefined || status === 'fresh';
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
/**
|
|
675
|
+
* The shape the store step relies on. It deletes the 30-day window before inserting, so
|
|
676
|
+
* an answer missing its lists (an error object that somehow exited 0) must never reach
|
|
677
|
+
* it: that would wipe the window and store nothing.
|
|
678
|
+
*/
|
|
679
|
+
function wellFormed(cost, usage) {
|
|
680
|
+
return cost?.ok !== false && usage?.ok !== false
|
|
681
|
+
&& Array.isArray(cost?.daily) && Array.isArray(usage?.sessions)
|
|
682
|
+
&& usage?.aggregates !== null && typeof usage?.aggregates === 'object';
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
async function collectAgentCosts(agent) {
|
|
686
|
+
const endDate = utcDate(Date.now());
|
|
687
|
+
const startDate = utcDate(Date.now() - (COSTS_WINDOW_DAYS - 1) * 86_400_000);
|
|
688
|
+
const range = { agentScope: 'all', startDate, endDate };
|
|
689
|
+
let cost = null;
|
|
690
|
+
let usage = null;
|
|
691
|
+
let status = null;
|
|
692
|
+
for (let attempt = 0; attempt < COSTS_FRESH_ATTEMPTS; attempt++) {
|
|
693
|
+
if (attempt) await new Promise((r) => setTimeout(r, COSTS_FRESH_WAIT_MS));
|
|
694
|
+
cost = await gatewayCall(agent.container, 'usage.cost', range);
|
|
695
|
+
usage = await gatewayCall(agent.container, 'sessions.usage', { ...range, limit: COSTS_SESSIONS_LIMIT });
|
|
696
|
+
if (isFresh(cost) && isFresh(usage)) break;
|
|
697
|
+
status = (isFresh(cost) ? usage : cost)?.cacheStatus?.status ?? null;
|
|
698
|
+
cost = null;
|
|
699
|
+
}
|
|
700
|
+
if (!cost) {
|
|
701
|
+
throw new Error(`OpenClaw's usage index still "${status}" after ${COSTS_FRESH_ATTEMPTS} tries; the previous figures are kept`);
|
|
702
|
+
}
|
|
703
|
+
if (!wellFormed(cost, usage)) throw new Error('unexpected answer from OpenClaw; the previous figures are kept');
|
|
704
|
+
|
|
705
|
+
const container = agent.container;
|
|
706
|
+
costs.db.transaction(() => {
|
|
707
|
+
costs.clearDaily.run(container, startDate);
|
|
708
|
+
for (const d of Array.isArray(cost.daily) ? cost.daily : []) {
|
|
709
|
+
if (!num(d.totalTokens) && !num(d.missingCostEntries) && !num(d.totalCost)) continue;
|
|
710
|
+
costs.insertDaily.run({
|
|
711
|
+
container, date: String(d.date),
|
|
712
|
+
input: num(d.input), output: num(d.output), cache_read: num(d.cacheRead), cache_write: num(d.cacheWrite),
|
|
713
|
+
total_tokens: num(d.totalTokens), cost: num(d.totalCost), input_cost: num(d.inputCost),
|
|
714
|
+
output_cost: num(d.outputCost), cache_read_cost: num(d.cacheReadCost), cache_write_cost: num(d.cacheWriteCost),
|
|
715
|
+
missing: num(d.missingCostEntries),
|
|
716
|
+
});
|
|
717
|
+
}
|
|
718
|
+
costs.clearModelDaily.run(container, startDate);
|
|
719
|
+
for (const m of Array.isArray(usage.aggregates?.modelDaily) ? usage.aggregates.modelDaily : []) {
|
|
720
|
+
costs.insertModelDaily.run({
|
|
721
|
+
container, date: String(m.date), provider: String(m.provider ?? 'unknown'), model: String(m.model ?? 'unknown'),
|
|
722
|
+
tokens: num(m.tokens), cost: num(m.cost), calls: num(m.count),
|
|
723
|
+
});
|
|
724
|
+
}
|
|
725
|
+
for (const s of Array.isArray(usage.sessions) ? usage.sessions : []) {
|
|
726
|
+
const u = s.usage || {};
|
|
727
|
+
const id = s.sessionId || u.sessionId || s.key;
|
|
728
|
+
if (!id) continue;
|
|
729
|
+
costs.upsertSession.run({
|
|
730
|
+
container, session_id: String(id), session_key: s.key ?? null, label: s.label ?? null,
|
|
731
|
+
agent_id: s.agentId ?? null, provider: s.modelProvider ?? null, model: s.model ?? null,
|
|
732
|
+
tokens: num(u.totalTokens), cost: num(u.totalCost), missing: num(u.missingCostEntries),
|
|
733
|
+
first_activity: u.firstActivity ?? null, last_activity: u.lastActivity ?? s.updatedAt ?? null,
|
|
734
|
+
});
|
|
735
|
+
}
|
|
736
|
+
costs.upsertSource.run({
|
|
737
|
+
container, agent_id: agent.agentId, name: agent.name, collected_at: Date.now(), attempted_at: Date.now(),
|
|
738
|
+
error: null, missing_by_model: JSON.stringify(cost.totals?.missingCostByModel || {}),
|
|
739
|
+
});
|
|
740
|
+
})();
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
// ── Vendor balances ──
|
|
744
|
+
//
|
|
745
|
+
// The check against the bill: what the vendor says it charged, next to what the agents'
|
|
746
|
+
// OpenClaw priced, over the same interval. Right after a pass of collectCosts — at most
|
|
747
|
+
// once an hour — the daemon reads each vendor's own figure and, in the same row, the
|
|
748
|
+
// agents' cumulative priced total for that vendor's models. Between two rows the balance
|
|
749
|
+
// drop (DeepSeek) or the usage growth (OpenRouter) is what the vendor billed, and the
|
|
750
|
+
// agents_cost growth is what the agents priced. Other users of the same key (the
|
|
751
|
+
// assistant, clients outside Rev4a) and top-ups show up as a difference; the API names
|
|
752
|
+
// both. Read-only calls, never billed.
|
|
753
|
+
|
|
754
|
+
const BALANCE_INTERVAL_MS = 60 * 60 * 1000;
|
|
755
|
+
const BALANCE_RETENTION_MS = 400 * 24 * 3600 * 1000;
|
|
756
|
+
const PROVIDER_KEYS_PATH = path.join(DATA_DIR, 'data', 'provider-keys.json');
|
|
757
|
+
|
|
758
|
+
const VENDOR_BALANCES = {
|
|
759
|
+
/** { currency, balance } — DeepSeek's total balance (topped-up plus granted). */
|
|
760
|
+
deepseek: async (key) => {
|
|
761
|
+
const r = await vendorFetch('https://api.deepseek.com/user/balance', key);
|
|
762
|
+
if (!r.ok) throw new Error(`DeepSeek balance API returned ${r.status}`);
|
|
763
|
+
const j = await r.json();
|
|
764
|
+
const info = (j.balance_infos || []).find((i) => i.currency === 'USD') || (j.balance_infos || [])[0];
|
|
765
|
+
if (!info) throw new Error('DeepSeek returned no balance');
|
|
766
|
+
return { currency: info.currency, balance: Number(info.total_balance), usage: null };
|
|
767
|
+
},
|
|
768
|
+
/** OpenRouter's lifetime usage, which only grows: the cleanest measure of spend. */
|
|
769
|
+
openrouter: async (key) => {
|
|
770
|
+
const r = await vendorFetch('https://openrouter.ai/api/v1/credits', key);
|
|
771
|
+
if (!r.ok) throw new Error(`OpenRouter credits API returned ${r.status}`);
|
|
772
|
+
const d = (await r.json()).data || {};
|
|
773
|
+
return { currency: 'USD', balance: Number(d.total_credits) - Number(d.total_usage), usage: Number(d.total_usage) };
|
|
774
|
+
},
|
|
775
|
+
};
|
|
776
|
+
|
|
777
|
+
/**
|
|
778
|
+
* A request that failed before any HTTP answer (network, invalid header) keeps only its
|
|
779
|
+
* kind: Node's message for an invalid header value quotes the header — the key itself,
|
|
780
|
+
* when a pasted key carries a line break.
|
|
781
|
+
*/
|
|
782
|
+
async function vendorFetch(url, key) {
|
|
783
|
+
try {
|
|
784
|
+
return await fetch(url, { headers: { Authorization: `Bearer ${key}` }, signal: AbortSignal.timeout(15_000) });
|
|
785
|
+
} catch (e) {
|
|
786
|
+
throw new Error(e?.name === 'TimeoutError' ? 'request timed out' : 'request failed (network, or a malformed key)');
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
async function collectVendorBalances() {
|
|
791
|
+
let keys = {};
|
|
792
|
+
try { keys = JSON.parse(fs.readFileSync(PROVIDER_KEYS_PATH, 'utf8')); } catch { return; }
|
|
793
|
+
const ts = Date.now();
|
|
794
|
+
for (const [vendor, read] of Object.entries(VENDOR_BALANCES)) {
|
|
795
|
+
if (!keys[vendor]) continue;
|
|
796
|
+
// Per vendor: one that failed last time is asked again next pass, not next hour.
|
|
797
|
+
const last = costs.lastBalanceTs.get(vendor).ts;
|
|
798
|
+
if (last && Date.now() - last < BALANCE_INTERVAL_MS - 60_000) continue;
|
|
799
|
+
try {
|
|
800
|
+
const b = await read(keys[vendor]);
|
|
801
|
+
if (!Number.isFinite(b.balance)) throw new Error('no numeric balance');
|
|
802
|
+
costs.insertBalance.run({ ts, vendor, currency: b.currency, balance: b.balance, usage: b.usage,
|
|
803
|
+
agents_cost: costs.agentsCostFor.get(vendor).cost });
|
|
804
|
+
} catch (e) {
|
|
805
|
+
log(`[COSTS] ${vendor} balance: ${e.message}`);
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
costs.pruneBalances.run(ts - BALANCE_RETENTION_MS);
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
let costsRunning = false;
|
|
812
|
+
async function collectCosts() {
|
|
813
|
+
if (!costs || costsRunning) return;
|
|
814
|
+
costsRunning = true;
|
|
815
|
+
try {
|
|
816
|
+
let agents;
|
|
817
|
+
try {
|
|
818
|
+
agents = await listRunningAgents();
|
|
819
|
+
} catch (e) {
|
|
820
|
+
log(`[COSTS] Docker unreachable: ${e.message}`);
|
|
821
|
+
return;
|
|
822
|
+
}
|
|
823
|
+
// Wake every agent first: the first call starts its index rebuild, and the rebuilds
|
|
824
|
+
// then run in parallel (about a minute each) instead of one after the other — after a
|
|
825
|
+
// price change every agent rebuilds, and in turn a pass would outlast the interval.
|
|
826
|
+
for (const agent of agents) {
|
|
827
|
+
await gatewayCall(agent.container, 'usage.cost', { agentScope: 'all', startDate: utcDate(Date.now()), endDate: utcDate(Date.now()) })
|
|
828
|
+
.catch(() => { /* the read below reports it */ });
|
|
829
|
+
}
|
|
830
|
+
// Then one agent at a time: each call is a short-lived CLI in that container.
|
|
831
|
+
for (const agent of agents) {
|
|
832
|
+
try {
|
|
833
|
+
await collectAgentCosts(agent);
|
|
834
|
+
} catch (e) {
|
|
835
|
+
costs.upsertSource.run({
|
|
836
|
+
container: agent.container, agent_id: agent.agentId, name: agent.name, collected_at: null,
|
|
837
|
+
attempted_at: Date.now(), error: String(e.message).slice(0, 300), missing_by_model: null,
|
|
838
|
+
});
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
// Only after the agents were read, so both sides of the row describe the same moment.
|
|
842
|
+
await collectVendorBalances().catch((e) => log(`[COSTS] balances: ${e.message}`));
|
|
843
|
+
} finally {
|
|
844
|
+
costsRunning = false;
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
|
|
451
848
|
// ── Helpers ─────────────────────────────────────────────────────────────────
|
|
452
849
|
|
|
453
850
|
function log(msg) {
|
|
@@ -609,7 +1006,7 @@ function pollSessions() {
|
|
|
609
1006
|
const model = s.model || null;
|
|
610
1007
|
const tokens_in = s.inputTokens || 0;
|
|
611
1008
|
const tokens_out = s.outputTokens || 0;
|
|
612
|
-
const cost_usd =
|
|
1009
|
+
const cost_usd = 0;
|
|
613
1010
|
const started_at = s.updatedAt ? s.updatedAt - (s.ageMs || 0) : now;
|
|
614
1011
|
const updated_at = s.updatedAt || now;
|
|
615
1012
|
|
|
@@ -660,14 +1057,14 @@ function pollSessions() {
|
|
|
660
1057
|
ts: now,
|
|
661
1058
|
session_id,
|
|
662
1059
|
type: 'complete',
|
|
663
|
-
data: JSON.stringify({
|
|
1060
|
+
data: JSON.stringify({ tokens: tokens_in + tokens_out }),
|
|
664
1061
|
});
|
|
665
1062
|
} else if (prevSnapshot && prevSnapshot.status !== 'error' && status === 'error') {
|
|
666
1063
|
pendingEvents.push({
|
|
667
1064
|
ts: now,
|
|
668
1065
|
session_id,
|
|
669
1066
|
type: 'fail',
|
|
670
|
-
data: JSON.stringify({
|
|
1067
|
+
data: JSON.stringify({ tokens: tokens_in + tokens_out, reason: 'openclaw_error_signal' }),
|
|
671
1068
|
});
|
|
672
1069
|
}
|
|
673
1070
|
|
|
@@ -821,14 +1218,28 @@ const firstSample = metrics
|
|
|
821
1218
|
: null;
|
|
822
1219
|
if (metrics) log(`Machine metrics every ${METRICS_INTERVAL_MS / 1000}s into ${METRICS_DB_PATH}`);
|
|
823
1220
|
|
|
1221
|
+
// Agent costs: first pass shortly after start (agents may still be booting), then every
|
|
1222
|
+
// interval. Off when costs.db failed.
|
|
1223
|
+
function sampleCosts() {
|
|
1224
|
+
collectCosts().catch((e) => log(`[COSTS ERROR] ${e.message}`));
|
|
1225
|
+
}
|
|
1226
|
+
let costsTimer = null;
|
|
1227
|
+
const firstCosts = costs
|
|
1228
|
+
? setTimeout(() => { sampleCosts(); costsTimer = setInterval(sampleCosts, COSTS_INTERVAL_MS); }, COSTS_FIRST_DELAY_MS)
|
|
1229
|
+
: null;
|
|
1230
|
+
if (costs) log(`Agent costs every ${COSTS_INTERVAL_MS / 60000} min into ${COSTS_DB_PATH}`);
|
|
1231
|
+
|
|
824
1232
|
// Graceful shutdown
|
|
825
1233
|
process.on('SIGTERM', () => {
|
|
826
1234
|
log('SIGTERM received, shutting down...');
|
|
827
1235
|
clearInterval(timer);
|
|
828
1236
|
clearTimeout(firstSample);
|
|
829
1237
|
clearInterval(metricsTimer);
|
|
1238
|
+
clearTimeout(firstCosts);
|
|
1239
|
+
clearInterval(costsTimer);
|
|
830
1240
|
db.close();
|
|
831
1241
|
metrics?.db.close();
|
|
1242
|
+
costs?.db.close();
|
|
832
1243
|
process.exit(0);
|
|
833
1244
|
});
|
|
834
1245
|
|
|
@@ -837,7 +1248,10 @@ process.on('SIGINT', () => {
|
|
|
837
1248
|
clearInterval(timer);
|
|
838
1249
|
clearTimeout(firstSample);
|
|
839
1250
|
clearInterval(metricsTimer);
|
|
1251
|
+
clearTimeout(firstCosts);
|
|
1252
|
+
clearInterval(costsTimer);
|
|
840
1253
|
db.close();
|
|
841
1254
|
metrics?.db.close();
|
|
1255
|
+
costs?.db.close();
|
|
842
1256
|
process.exit(0);
|
|
843
1257
|
});
|
package/docs/ARCHITECTURE.md
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Rev4a Architecture — Design & Vision
|
|
2
2
|
|
|
3
3
|
> **Status:** Active — `main` branch
|
|
4
|
-
> **Last updated:** 2026-09-
|
|
4
|
+
> **Last updated:** 2026-09-27
|
|
5
5
|
> **Goal:** Transform Rev4a from a monitoring dashboard into a central orchestrator for a distributed multi-container agency.
|
|
6
6
|
|
|
7
7
|
---
|
|
@@ -136,7 +136,8 @@ The central container, running the Next.js dashboard + orchestration API.
|
|
|
136
136
|
hand (identity, `enabled`, `deprecated`, and an optional `info` block for what no API
|
|
137
137
|
publishes: the vendor's size claim, benchmarks, the docs link, notes).
|
|
138
138
|
`model-pricing.json` and `model-details.json` are **generated** by
|
|
139
|
-
`npm run refresh:pricing` (`scripts/refresh-model-pricing.mjs`): prices from OpenRouter
|
|
139
|
+
`npm run refresh:pricing` (`scripts/refresh-model-pricing.mjs`): prices from OpenRouter
|
|
140
|
+
(cache rates included; direct vendors' prices by hand from their pricing pages),
|
|
140
141
|
and per model the description, architecture and benchmarks from OpenRouter plus the size,
|
|
141
142
|
weight mix, licence and dates from the Hugging Face card. Two scripts back the curation
|
|
142
143
|
of `info`: `npm run info:suggest` (`scripts/model-info-suggest.mjs`) prints the candidate
|
|
@@ -578,7 +579,9 @@ The Rev4a daemon (`daemon.js`) is a standalone Node.js process that bridges the
|
|
|
578
579
|
4. Sample the host machine (CPU, RAM, swap, storage) every 30 s into `metrics.db`, on a
|
|
579
580
|
timer of its own
|
|
580
581
|
5. Detect anomalies (CPU > 85%, RAM > 90%, a disk > 90%) and record them as events
|
|
581
|
-
6.
|
|
582
|
+
6. Read what each running agent spent, as its own OpenClaw priced it, every 5 min into
|
|
583
|
+
`costs.db`, and each vendor's balance or usage every hour (see [Agent Costs](#agent-costs))
|
|
584
|
+
7. Manage DB lifecycle (WAL mode; events.db checkpointed after each poll cycle)
|
|
582
585
|
|
|
583
586
|
### Poll Interval
|
|
584
587
|
|
|
@@ -586,33 +589,12 @@ A fixed 30 s timer, whether or not sessions are working. When the OpenClaw CLI i
|
|
|
586
589
|
the host, the session poll logs it once and stays idle; the machine metrics keep being
|
|
587
590
|
sampled, since they have their own timer.
|
|
588
591
|
|
|
589
|
-
###
|
|
592
|
+
### Session costs
|
|
590
593
|
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
| `claude-sonnet-4` | substring | $3.00 | $15.00 |
|
|
596
|
-
| `claude-opus-4` | substring | $15.00 | $75.00 |
|
|
597
|
-
| `gpt-5-mini` | substring | $0.15 | $0.60 |
|
|
598
|
-
| `codex` | substring | $3.00 | $15.00 |
|
|
599
|
-
| `gemini` | substring | $0.075 | $0.30 |
|
|
600
|
-
| `flash` | substring | $0.075 | $0.30 |
|
|
601
|
-
| `deepseek` | substring | $0.55 | $2.19 |
|
|
602
|
-
| `default` | fallback | $3.00 | $15.00 |
|
|
603
|
-
|
|
604
|
-
**Aliases** (resolved before substring match):
|
|
605
|
-
|
|
606
|
-
| Alias | Resolves to |
|
|
607
|
-
|---|---|
|
|
608
|
-
| `cheap` | `flash` |
|
|
609
|
-
| `fast` | `claude-sonnet-4` |
|
|
610
|
-
| `big` | `claude-opus-4` |
|
|
611
|
-
| `coder` | `codex` |
|
|
612
|
-
| `pro` | `gemini` |
|
|
613
|
-
| `reason` | `deepseek` |
|
|
614
|
-
|
|
615
|
-
Formula: `cost = (tokens_in / 1_000_000 * price_in) + (tokens_out / 1_000_000 * price_out)`
|
|
594
|
+
The session poll writes no cost: `sessions.cost_usd` stays 0. It used to be estimated from a
|
|
595
|
+
fixed 2025 rate table (`$3 / $15` per 1M for anything unknown), which the dashboard showed as
|
|
596
|
+
spend. What the agents actually spend is priced by each agent's own OpenClaw and read by the
|
|
597
|
+
Agent Costs timer — see [Agent Costs](#agent-costs).
|
|
616
598
|
|
|
617
599
|
### Upsert Logic
|
|
618
600
|
|
|
@@ -647,6 +629,34 @@ taken at load), then one every 30 s. Samples older than 30 days are pruned at ev
|
|
|
647
629
|
it through `lib/metrics-db.ts` (read-only, `null` when there is nothing to read);
|
|
648
630
|
`GET /api/metrics` serves the System page (`/system`) and the dashboard's Machine card.
|
|
649
631
|
|
|
632
|
+
### Agent Costs
|
|
633
|
+
|
|
634
|
+
Rev4a computes no cost. Every model in the `rev4a` provider block synced into the agents
|
|
635
|
+
carries its price (`cost`, from `model-pricing.json` through `priceAt()`), so each agent's
|
|
636
|
+
OpenClaw prices every call itself — per call, session and day, cache split out. A vendor
|
|
637
|
+
priced by time of day (DeepSeek off-peak) is re-synced at each change of band by
|
|
638
|
+
`lib/price-schedule-sync.ts`, started from `instrumentation.ts`; OpenClaw keeps the cost
|
|
639
|
+
recorded at call time. Details in `docs/dev/GATEWAY.md` (Costs).
|
|
640
|
+
|
|
641
|
+
Every 5 min (first pass 20 s after start) the daemon first wakes every running agent with one
|
|
642
|
+
short call — OpenClaw rebuilds its transcript index in about a minute after a restart or a
|
|
643
|
+
price change, and woken together the agents rebuild in parallel — then asks each, one at a
|
|
644
|
+
time, for its figures over the last 30 UTC days — `usage.cost` and
|
|
645
|
+
`sessions.usage` through `docker exec … openclaw gateway call` on the agent's port 3000,
|
|
646
|
+
the token resolved inside the container — and rewrites that window in `costs.db` (schema
|
|
647
|
+
in `docs/dev/DATABASE.md`). An answer from an index OpenClaw is still rebuilding is never
|
|
648
|
+
stored. The readers — `GET /api/costs/usage` (the Costs page, `/costs`), `GET /api/costs` and
|
|
649
|
+
the SSE stream (the dashboard), the system-health cost check, and `GET /api/costs/agent` (the
|
|
650
|
+
agent panel) — share one read-only handle (`lib/costs-db.ts`) and one set of spend queries
|
|
651
|
+
(`lib/agent-costs.ts`); the vendor readings are read by `lib/cost-reconciliation.ts`.
|
|
652
|
+
|
|
653
|
+
After a pass, at most once an hour, the daemon also reads each vendor's own figure —
|
|
654
|
+
DeepSeek's balance, OpenRouter's lifetime usage (read-only calls, never billed) — and stores
|
|
655
|
+
it next to the agents' cumulative priced total for that vendor's models, in the same row
|
|
656
|
+
(`vendor_balance`). Between two rows, what the vendor billed and what the agents priced
|
|
657
|
+
cover the same interval: the Costs page's *Against the bill* (`lib/cost-reconciliation.ts`).
|
|
658
|
+
`costs.db` is opened in a try: if it cannot be, collection is off (`[COSTS] disabled`).
|
|
659
|
+
|
|
650
660
|
### Anomaly Detection
|
|
651
661
|
|
|
652
662
|
After each sample the daemon inserts a `system_anomaly` event into `events.db`
|