@aion0/forge 0.4.14 → 0.4.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +1 -1
- package/README.md +1 -1
- package/RELEASE_NOTES.md +19 -11
- package/app/api/mobile-chat/route.ts +23 -1
- package/app/api/usage/route.ts +20 -0
- package/bin/forge-server.mjs +10 -3
- package/cli/mw.ts +2 -2
- package/components/Dashboard.tsx +17 -1
- package/components/DocTerminal.tsx +2 -2
- package/components/HelpTerminal.tsx +2 -2
- package/components/SkillsPanel.tsx +9 -0
- package/components/UsagePanel.tsx +207 -0
- package/components/WebTerminal.tsx +2 -2
- package/docs/LOCAL-DEPLOY.md +15 -15
- package/lib/cloudflared.ts +1 -1
- package/lib/help-docs/00-overview.md +1 -1
- package/lib/help-docs/05-pipelines.md +6 -6
- package/lib/init.ts +9 -2
- package/lib/task-manager.ts +26 -1
- package/lib/telegram-standalone.ts +1 -1
- package/lib/terminal-server.ts +2 -2
- package/lib/terminal-standalone.ts +1 -1
- package/lib/usage-scanner.ts +249 -0
- package/next-env.d.ts +1 -1
- package/package.json +2 -2
- package/scripts/verify-usage.ts +178 -0
- package/src/config/index.ts +1 -1
- package/src/core/db/database.ts +28 -0
- package/start.sh +3 -0
package/lib/init.ts
CHANGED
|
@@ -106,6 +106,13 @@ export function ensureInitialized() {
|
|
|
106
106
|
setInterval(() => { syncSkills().catch(() => {}); }, 60 * 60 * 1000);
|
|
107
107
|
} catch {}
|
|
108
108
|
|
|
109
|
+
// Usage scanner — scan JSONL files for token usage on startup + every hour
|
|
110
|
+
try {
|
|
111
|
+
const { scanUsage } = require('./usage-scanner');
|
|
112
|
+
scanUsage();
|
|
113
|
+
setInterval(() => { try { scanUsage(); } catch {} }, 60 * 60 * 1000);
|
|
114
|
+
} catch {}
|
|
115
|
+
|
|
109
116
|
// Task runner is safe in every worker (DB-level coordination)
|
|
110
117
|
ensureRunnerStarted();
|
|
111
118
|
|
|
@@ -175,7 +182,7 @@ function startTelegramProcess() {
|
|
|
175
182
|
const script = join(process.cwd(), 'lib', 'telegram-standalone.ts');
|
|
176
183
|
telegramChild = spawn('npx', ['tsx', script], {
|
|
177
184
|
stdio: ['ignore', 'inherit', 'inherit'],
|
|
178
|
-
env: { ...process.env, PORT: String(process.env.PORT ||
|
|
185
|
+
env: { ...process.env, PORT: String(process.env.PORT || 8403) },
|
|
179
186
|
detached: false,
|
|
180
187
|
});
|
|
181
188
|
telegramChild.on('exit', () => { telegramChild = null; });
|
|
@@ -187,7 +194,7 @@ let terminalChild: ReturnType<typeof spawn> | null = null;
|
|
|
187
194
|
function startTerminalProcess() {
|
|
188
195
|
if (terminalChild) return;
|
|
189
196
|
|
|
190
|
-
const termPort = Number(process.env.TERMINAL_PORT) ||
|
|
197
|
+
const termPort = Number(process.env.TERMINAL_PORT) || 8404;
|
|
191
198
|
|
|
192
199
|
const net = require('node:net');
|
|
193
200
|
const tester = net.createServer();
|
package/lib/task-manager.ts
CHANGED
|
@@ -322,6 +322,8 @@ function executeTask(task: Task): Promise<void> {
|
|
|
322
322
|
let totalCost = 0;
|
|
323
323
|
let sessionId = '';
|
|
324
324
|
let modelUsed = '';
|
|
325
|
+
let totalInputTokens = 0;
|
|
326
|
+
let totalOutputTokens = 0;
|
|
325
327
|
|
|
326
328
|
child.on('error', (err) => {
|
|
327
329
|
console.error(`[task-runner] Spawn error:`, err.message);
|
|
@@ -355,9 +357,16 @@ function executeTask(task: Task): Promise<void> {
|
|
|
355
357
|
if (parsed.type === 'system' && parsed.subtype === 'init' && parsed.model) {
|
|
356
358
|
modelUsed = parsed.model;
|
|
357
359
|
}
|
|
360
|
+
// Accumulate token usage from assistant messages
|
|
361
|
+
if (parsed.type === 'assistant' && parsed.message?.usage) {
|
|
362
|
+
totalInputTokens += parsed.message.usage.input_tokens || 0;
|
|
363
|
+
totalOutputTokens += parsed.message.usage.output_tokens || 0;
|
|
364
|
+
}
|
|
358
365
|
if (parsed.type === 'result') {
|
|
359
366
|
resultText = typeof parsed.result === 'string' ? parsed.result : JSON.stringify(parsed.result);
|
|
360
367
|
totalCost = parsed.total_cost_usd || 0;
|
|
368
|
+
if (parsed.total_input_tokens) totalInputTokens = parsed.total_input_tokens;
|
|
369
|
+
if (parsed.total_output_tokens) totalOutputTokens = parsed.total_output_tokens;
|
|
361
370
|
}
|
|
362
371
|
} catch {}
|
|
363
372
|
}
|
|
@@ -412,7 +421,23 @@ function executeTask(task: Task): Promise<void> {
|
|
|
412
421
|
WHERE id = ?
|
|
413
422
|
`).run(resultText, totalCost, task.id);
|
|
414
423
|
emit(task.id, 'status', 'done');
|
|
415
|
-
console.log(`[task] Done: ${task.id} ${task.projectName} (cost: $${totalCost?.toFixed(4) || '0'})`);
|
|
424
|
+
console.log(`[task] Done: ${task.id} ${task.projectName} (cost: $${totalCost?.toFixed(4) || '0'}, ${totalInputTokens}in/${totalOutputTokens}out)`);
|
|
425
|
+
// Record usage
|
|
426
|
+
try {
|
|
427
|
+
const { recordUsage } = require('./usage-scanner');
|
|
428
|
+
let isPipeline = false;
|
|
429
|
+
try { const { pipelineTaskIds: ptids } = require('./pipeline'); isPipeline = ptids.has(task.id); } catch {}
|
|
430
|
+
recordUsage({
|
|
431
|
+
sessionId: sessionId || task.id,
|
|
432
|
+
source: isPipeline ? 'pipeline' : 'task',
|
|
433
|
+
projectPath: task.projectPath,
|
|
434
|
+
projectName: task.projectName,
|
|
435
|
+
model: modelUsed || 'unknown',
|
|
436
|
+
inputTokens: totalInputTokens,
|
|
437
|
+
outputTokens: totalOutputTokens,
|
|
438
|
+
taskId: task.id,
|
|
439
|
+
});
|
|
440
|
+
} catch {}
|
|
416
441
|
const doneTask = getTask(task.id);
|
|
417
442
|
if (doneTask) notifyTaskComplete(doneTask).catch(() => {});
|
|
418
443
|
notifyTerminalSession(task, 'done', sessionId);
|
|
@@ -61,7 +61,7 @@ async function poll() {
|
|
|
61
61
|
|
|
62
62
|
// Forward to Next.js API for processing
|
|
63
63
|
try {
|
|
64
|
-
await fetch(`http://localhost:${process.env.PORT ||
|
|
64
|
+
await fetch(`http://localhost:${process.env.PORT || 8403}/api/telegram`, {
|
|
65
65
|
method: 'POST',
|
|
66
66
|
headers: { 'Content-Type': 'application/json', 'x-telegram-secret': TOKEN },
|
|
67
67
|
body: JSON.stringify(update.message),
|
package/lib/terminal-server.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Terminal Server — standalone WebSocket PTY server.
|
|
3
|
-
* Runs on port
|
|
3
|
+
* Runs on port 8404 alongside the Next.js server on 8403.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import { WebSocketServer, WebSocket } from 'ws';
|
|
@@ -9,7 +9,7 @@ import { homedir } from 'node:os';
|
|
|
9
9
|
|
|
10
10
|
let wss: WebSocketServer | null = null;
|
|
11
11
|
|
|
12
|
-
export function startTerminalServer(port =
|
|
12
|
+
export function startTerminalServer(port = 8404) {
|
|
13
13
|
if (wss) return;
|
|
14
14
|
|
|
15
15
|
wss = new WebSocketServer({ port });
|
|
@@ -33,7 +33,7 @@ import { getDataDir } from './dirs';
|
|
|
33
33
|
import { readFileSync, writeFileSync, mkdirSync, readdirSync } from 'node:fs';
|
|
34
34
|
import { join } from 'node:path';
|
|
35
35
|
|
|
36
|
-
const PORT = Number(process.env.TERMINAL_PORT) ||
|
|
36
|
+
const PORT = Number(process.env.TERMINAL_PORT) || 8404;
|
|
37
37
|
// Session prefix based on DATA_DIR hash — default instance keeps 'mw-' for backward compat
|
|
38
38
|
const _dataDir = process.env.FORGE_DATA_DIR || '';
|
|
39
39
|
const _isDefault = !_dataDir || _dataDir.endsWith('/data') || _dataDir.endsWith('/.forge');
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Usage Scanner — scans Claude Code JSONL session files for token usage data.
|
|
3
|
+
* Stores per-day aggregated results in SQLite for accurate daily breakdown.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { readdirSync, readFileSync, statSync } from 'node:fs';
|
|
7
|
+
import { join, basename } from 'node:path';
|
|
8
|
+
import { getDb } from '@/src/core/db/database';
|
|
9
|
+
import { getDbPath } from '@/src/config';
|
|
10
|
+
import { getClaudeDir } from './dirs';
|
|
11
|
+
|
|
12
|
+
function db() { return getDb(getDbPath()); }
|
|
13
|
+
|
|
14
|
+
const PRICING: Record<string, { input: number; output: number }> = {
|
|
15
|
+
'claude-opus-4': { input: 15, output: 75 },
|
|
16
|
+
'claude-sonnet-4': { input: 3, output: 15 },
|
|
17
|
+
'claude-haiku-4': { input: 0.80, output: 4 },
|
|
18
|
+
'default': { input: 3, output: 15 },
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
function getModelFamily(model: string): string {
|
|
22
|
+
if (!model) return 'unknown';
|
|
23
|
+
if (model.includes('opus')) return 'claude-opus-4';
|
|
24
|
+
if (model.includes('haiku')) return 'claude-haiku-4';
|
|
25
|
+
if (model.includes('sonnet')) return 'claude-sonnet-4';
|
|
26
|
+
return 'unknown';
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function calcCost(family: string, input: number, output: number): number {
|
|
30
|
+
const p = PRICING[family] || PRICING['default'];
|
|
31
|
+
// Only count input + output tokens. Cache tokens excluded from cost estimate
|
|
32
|
+
// because subscriptions (Max/Pro) don't charge per-token for cache.
|
|
33
|
+
return (input * p.input / 1_000_000) + (output * p.output / 1_000_000);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function dirToProjectPath(dirName: string): string {
|
|
37
|
+
return dirName.replace(/^-/, '/').replace(/-/g, '/');
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function dirToProjectName(dirName: string): string {
|
|
41
|
+
return dirToProjectPath(dirName).split('/').pop() || dirName;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Get local date string from UTC timestamp */
|
|
45
|
+
function toLocalDate(ts: string): string {
|
|
46
|
+
if (!ts) return 'unknown';
|
|
47
|
+
try {
|
|
48
|
+
const d = new Date(ts);
|
|
49
|
+
return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`;
|
|
50
|
+
} catch {
|
|
51
|
+
return ts.slice(0, 10) || 'unknown';
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
interface DayModelBucket {
|
|
56
|
+
input: number; output: number; cacheRead: number; cacheCreate: number; count: number;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Parse JSONL and aggregate by day + model */
|
|
60
|
+
function parseByDayModel(content: string): Map<string, DayModelBucket> {
|
|
61
|
+
// key: "day|model"
|
|
62
|
+
const buckets = new Map<string, DayModelBucket>();
|
|
63
|
+
|
|
64
|
+
for (const line of content.split('\n')) {
|
|
65
|
+
if (!line.trim()) continue;
|
|
66
|
+
try {
|
|
67
|
+
const obj = JSON.parse(line);
|
|
68
|
+
if (obj.type === 'assistant' && obj.message?.usage) {
|
|
69
|
+
const u = obj.message.usage;
|
|
70
|
+
const model = getModelFamily(obj.message.model || '');
|
|
71
|
+
const day = toLocalDate(obj.timestamp || '');
|
|
72
|
+
const key = `${day}|${model}`;
|
|
73
|
+
|
|
74
|
+
let b = buckets.get(key);
|
|
75
|
+
if (!b) { b = { input: 0, output: 0, cacheRead: 0, cacheCreate: 0, count: 0 }; buckets.set(key, b); }
|
|
76
|
+
b.input += u.input_tokens || 0;
|
|
77
|
+
b.output += u.output_tokens || 0;
|
|
78
|
+
b.cacheRead += u.cache_read_input_tokens || 0;
|
|
79
|
+
b.cacheCreate += u.cache_creation_input_tokens || 0;
|
|
80
|
+
b.count++;
|
|
81
|
+
}
|
|
82
|
+
} catch {}
|
|
83
|
+
}
|
|
84
|
+
return buckets;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Scan all JSONL files */
|
|
88
|
+
export function scanUsage(): { scanned: number; updated: number; errors: number } {
|
|
89
|
+
const projectsDir = join(getClaudeDir(), 'projects');
|
|
90
|
+
let scanned = 0, updated = 0, errors = 0;
|
|
91
|
+
|
|
92
|
+
let projectDirs: string[];
|
|
93
|
+
try { projectDirs = readdirSync(projectsDir); } catch { return { scanned: 0, updated: 0, errors: 0 }; }
|
|
94
|
+
|
|
95
|
+
const upsert = db().prepare(`
|
|
96
|
+
INSERT INTO token_usage (session_id, source, project_path, project_name, model, day, input_tokens, output_tokens, cache_read_tokens, cache_create_tokens, cost_usd, message_count)
|
|
97
|
+
VALUES (?, 'terminal', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
98
|
+
ON CONFLICT(session_id, source, model, day) DO UPDATE SET
|
|
99
|
+
input_tokens = excluded.input_tokens, output_tokens = excluded.output_tokens,
|
|
100
|
+
cache_read_tokens = excluded.cache_read_tokens, cache_create_tokens = excluded.cache_create_tokens,
|
|
101
|
+
cost_usd = excluded.cost_usd, message_count = excluded.message_count
|
|
102
|
+
`);
|
|
103
|
+
|
|
104
|
+
const getScanState = db().prepare('SELECT last_size FROM usage_scan_state WHERE file_path = ?');
|
|
105
|
+
const setScanState = db().prepare(`
|
|
106
|
+
INSERT INTO usage_scan_state (file_path, last_size) VALUES (?, ?)
|
|
107
|
+
ON CONFLICT(file_path) DO UPDATE SET last_size = excluded.last_size, last_scan = datetime('now')
|
|
108
|
+
`);
|
|
109
|
+
|
|
110
|
+
for (const projDir of projectDirs) {
|
|
111
|
+
const projPath = join(projectsDir, projDir);
|
|
112
|
+
try { if (!statSync(projPath).isDirectory()) continue; } catch { continue; }
|
|
113
|
+
|
|
114
|
+
const projectPath = dirToProjectPath(projDir);
|
|
115
|
+
const projectName = dirToProjectName(projDir);
|
|
116
|
+
let files: string[];
|
|
117
|
+
try { files = readdirSync(projPath).filter(f => f.endsWith('.jsonl') && !f.startsWith('agent-')); } catch { continue; }
|
|
118
|
+
|
|
119
|
+
for (const file of files) {
|
|
120
|
+
const filePath = join(projPath, file);
|
|
121
|
+
const sessionId = basename(file, '.jsonl');
|
|
122
|
+
scanned++;
|
|
123
|
+
|
|
124
|
+
try {
|
|
125
|
+
const currentSize = statSync(filePath).size;
|
|
126
|
+
const scanState = getScanState.get(filePath) as { last_size: number } | undefined;
|
|
127
|
+
if (currentSize === (scanState?.last_size || 0)) continue;
|
|
128
|
+
|
|
129
|
+
const content = readFileSync(filePath, 'utf-8');
|
|
130
|
+
const buckets = parseByDayModel(content);
|
|
131
|
+
|
|
132
|
+
if (buckets.size === 0) { setScanState.run(filePath, currentSize); continue; }
|
|
133
|
+
|
|
134
|
+
for (const [key, b] of buckets) {
|
|
135
|
+
const [day, model] = key.split('|');
|
|
136
|
+
const cost = calcCost(model, b.input, b.output);
|
|
137
|
+
upsert.run(sessionId, projectPath, projectName, model, day, b.input, b.output, b.cacheRead, b.cacheCreate, cost, b.count);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
setScanState.run(filePath, currentSize);
|
|
141
|
+
updated++;
|
|
142
|
+
} catch { errors++; }
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return { scanned, updated, errors };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** Record usage from task/mobile/pipeline */
|
|
149
|
+
export function recordUsage(opts: {
|
|
150
|
+
sessionId: string;
|
|
151
|
+
source: 'task' | 'mobile' | 'pipeline';
|
|
152
|
+
projectPath: string;
|
|
153
|
+
projectName: string;
|
|
154
|
+
model: string;
|
|
155
|
+
inputTokens: number;
|
|
156
|
+
outputTokens: number;
|
|
157
|
+
cacheReadTokens?: number;
|
|
158
|
+
cacheCreateTokens?: number;
|
|
159
|
+
taskId?: string;
|
|
160
|
+
}): void {
|
|
161
|
+
const family = getModelFamily(opts.model);
|
|
162
|
+
const cost = calcCost(family, opts.inputTokens, opts.outputTokens);
|
|
163
|
+
const day = toLocalDate(new Date().toISOString());
|
|
164
|
+
|
|
165
|
+
db().prepare(`
|
|
166
|
+
INSERT INTO token_usage (session_id, source, project_path, project_name, model, day, input_tokens, output_tokens, cache_read_tokens, cache_create_tokens, cost_usd, message_count, task_id)
|
|
167
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?)
|
|
168
|
+
ON CONFLICT(session_id, source, model, day) DO UPDATE SET
|
|
169
|
+
input_tokens = token_usage.input_tokens + excluded.input_tokens,
|
|
170
|
+
output_tokens = token_usage.output_tokens + excluded.output_tokens,
|
|
171
|
+
cost_usd = token_usage.cost_usd + excluded.cost_usd,
|
|
172
|
+
message_count = token_usage.message_count + 1
|
|
173
|
+
`).run(opts.sessionId, opts.source, opts.projectPath, opts.projectName, family, day, opts.inputTokens, opts.outputTokens, opts.cacheReadTokens || 0, opts.cacheCreateTokens || 0, cost, opts.taskId || null);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** Query usage data */
|
|
177
|
+
export function queryUsage(opts: {
|
|
178
|
+
days?: number;
|
|
179
|
+
projectName?: string;
|
|
180
|
+
source?: string;
|
|
181
|
+
model?: string;
|
|
182
|
+
}): {
|
|
183
|
+
total: { input: number; output: number; cost: number; sessions: number; messages: number };
|
|
184
|
+
byProject: { name: string; input: number; output: number; cost: number; sessions: number }[];
|
|
185
|
+
byModel: { model: string; input: number; output: number; cost: number; messages: number }[];
|
|
186
|
+
byDay: { date: string; input: number; output: number; cost: number }[];
|
|
187
|
+
bySource: { source: string; input: number; output: number; cost: number; messages: number }[];
|
|
188
|
+
} {
|
|
189
|
+
let where = '1=1';
|
|
190
|
+
const params: any[] = [];
|
|
191
|
+
|
|
192
|
+
if (opts.days) {
|
|
193
|
+
const cutoff = new Date();
|
|
194
|
+
cutoff.setDate(cutoff.getDate() - opts.days);
|
|
195
|
+
const cutoffDay = `${cutoff.getFullYear()}-${String(cutoff.getMonth() + 1).padStart(2, '0')}-${String(cutoff.getDate()).padStart(2, '0')}`;
|
|
196
|
+
where += ' AND day >= ?';
|
|
197
|
+
params.push(cutoffDay);
|
|
198
|
+
}
|
|
199
|
+
if (opts.projectName) { where += ' AND project_name = ?'; params.push(opts.projectName); }
|
|
200
|
+
if (opts.source) { where += ' AND source = ?'; params.push(opts.source); }
|
|
201
|
+
if (opts.model) { where += ' AND model = ?'; params.push(opts.model); }
|
|
202
|
+
|
|
203
|
+
const totalRow = db().prepare(`
|
|
204
|
+
SELECT COALESCE(SUM(input_tokens), 0) as input, COALESCE(SUM(output_tokens), 0) as output,
|
|
205
|
+
COALESCE(SUM(cost_usd), 0) as cost, COUNT(DISTINCT session_id) as sessions,
|
|
206
|
+
COALESCE(SUM(message_count), 0) as messages
|
|
207
|
+
FROM token_usage WHERE ${where}
|
|
208
|
+
`).get(...params) as any;
|
|
209
|
+
|
|
210
|
+
const byProject = (db().prepare(`
|
|
211
|
+
SELECT project_name as name, SUM(input_tokens) as input, SUM(output_tokens) as output,
|
|
212
|
+
SUM(cost_usd) as cost, COUNT(DISTINCT session_id) as sessions
|
|
213
|
+
FROM token_usage WHERE ${where}
|
|
214
|
+
GROUP BY project_name ORDER BY cost DESC LIMIT 20
|
|
215
|
+
`).all(...params) as any[]).map(r => ({
|
|
216
|
+
name: r.name, input: r.input, output: r.output, cost: +r.cost.toFixed(4), sessions: r.sessions,
|
|
217
|
+
}));
|
|
218
|
+
|
|
219
|
+
const byModel = (db().prepare(`
|
|
220
|
+
SELECT model, SUM(input_tokens) as input, SUM(output_tokens) as output,
|
|
221
|
+
SUM(cost_usd) as cost, SUM(message_count) as messages
|
|
222
|
+
FROM token_usage WHERE ${where}
|
|
223
|
+
GROUP BY model ORDER BY cost DESC
|
|
224
|
+
`).all(...params) as any[]).map(r => ({
|
|
225
|
+
model: r.model, input: r.input, output: r.output, cost: +r.cost.toFixed(4), messages: r.messages,
|
|
226
|
+
}));
|
|
227
|
+
|
|
228
|
+
const byDay = (db().prepare(`
|
|
229
|
+
SELECT day as date, SUM(input_tokens) as input, SUM(output_tokens) as output, SUM(cost_usd) as cost
|
|
230
|
+
FROM token_usage WHERE ${where} AND day != 'unknown'
|
|
231
|
+
GROUP BY day ORDER BY day DESC LIMIT 30
|
|
232
|
+
`).all(...params) as any[]).map(r => ({
|
|
233
|
+
date: r.date, input: r.input, output: r.output, cost: +r.cost.toFixed(4),
|
|
234
|
+
}));
|
|
235
|
+
|
|
236
|
+
const bySource = (db().prepare(`
|
|
237
|
+
SELECT source, SUM(input_tokens) as input, SUM(output_tokens) as output,
|
|
238
|
+
SUM(cost_usd) as cost, SUM(message_count) as messages
|
|
239
|
+
FROM token_usage WHERE ${where}
|
|
240
|
+
GROUP BY source ORDER BY cost DESC
|
|
241
|
+
`).all(...params) as any[]).map(r => ({
|
|
242
|
+
source: r.source, input: r.input, output: r.output, cost: +r.cost.toFixed(4), messages: r.messages,
|
|
243
|
+
}));
|
|
244
|
+
|
|
245
|
+
return {
|
|
246
|
+
total: { input: totalRow.input, output: totalRow.output, cost: +totalRow.cost.toFixed(4), sessions: totalRow.sessions, messages: totalRow.messages },
|
|
247
|
+
byProject, byModel, byDay, bySource,
|
|
248
|
+
};
|
|
249
|
+
}
|
package/next-env.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/// <reference types="next" />
|
|
2
2
|
/// <reference types="next/image-types/global" />
|
|
3
|
-
import "./.next/
|
|
3
|
+
import "./.next/types/routes.d.ts";
|
|
4
4
|
|
|
5
5
|
// NOTE: This file should not be edited
|
|
6
6
|
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aion0/forge",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.16",
|
|
4
4
|
"description": "Unified AI workflow platform — multi-model task orchestration, persistent sessions, web terminal, remote access",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"scripts": {
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
"@xyflow/react": "^12.10.1",
|
|
38
38
|
"ai": "^6.0.116",
|
|
39
39
|
"better-sqlite3": "^12.6.2",
|
|
40
|
-
"next": "^16.1
|
|
40
|
+
"next": "^16.2.1",
|
|
41
41
|
"next-auth": "5.0.0-beta.30",
|
|
42
42
|
"node-pty": "1.0.0",
|
|
43
43
|
"react": "^19.2.4",
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Verification script — compares direct JSONL scanning with DB scanner results.
|
|
3
|
+
* Run: npx tsx scripts/verify-usage.ts
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { readdirSync, readFileSync, statSync } from 'fs';
|
|
7
|
+
import { join, basename } from 'path';
|
|
8
|
+
import { homedir } from 'os';
|
|
9
|
+
|
|
10
|
+
const CLAUDE_DIR = join(homedir(), '.claude', 'projects');
|
|
11
|
+
|
|
12
|
+
const PRICING: Record<string, { input: number; output: number }> = {
|
|
13
|
+
'claude-opus-4': { input: 15, output: 75 },
|
|
14
|
+
'claude-sonnet-4': { input: 3, output: 15 },
|
|
15
|
+
'claude-haiku-4': { input: 0.80, output: 4 },
|
|
16
|
+
'default': { input: 3, output: 15 },
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
function getModelFamily(model: string): string {
|
|
20
|
+
if (!model) return 'unknown';
|
|
21
|
+
if (model.includes('opus')) return 'claude-opus-4';
|
|
22
|
+
if (model.includes('haiku')) return 'claude-haiku-4';
|
|
23
|
+
if (model.includes('sonnet')) return 'claude-sonnet-4';
|
|
24
|
+
return 'unknown';
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function calcCost(family: string, input: number, output: number, cacheRead: number, cacheCreate: number): number {
|
|
28
|
+
const p = PRICING[family] || PRICING['default'];
|
|
29
|
+
return (
|
|
30
|
+
(input * p.input / 1_000_000) +
|
|
31
|
+
(output * p.output / 1_000_000) +
|
|
32
|
+
(cacheRead * p.input * 0.1 / 1_000_000) +
|
|
33
|
+
(cacheCreate * p.input * 0.25 / 1_000_000)
|
|
34
|
+
);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
interface ProjectStats {
|
|
38
|
+
input: number; output: number; cost: number; sessions: number; messages: number;
|
|
39
|
+
cacheRead: number; cacheCreate: number;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
interface ModelStats {
|
|
43
|
+
input: number; output: number; cost: number; messages: number;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
interface DayStats {
|
|
47
|
+
input: number; output: number; cost: number;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const byProject: Record<string, ProjectStats> = {};
|
|
51
|
+
const byModel: Record<string, ModelStats> = {};
|
|
52
|
+
const byDay: Record<string, DayStats> = {};
|
|
53
|
+
let totalInput = 0, totalOutput = 0, totalCost = 0, totalSessions = 0, totalMessages = 0;
|
|
54
|
+
|
|
55
|
+
console.log('Scanning JSONL files...\n');
|
|
56
|
+
|
|
57
|
+
const projectDirs = readdirSync(CLAUDE_DIR);
|
|
58
|
+
let fileCount = 0;
|
|
59
|
+
|
|
60
|
+
for (const projDir of projectDirs) {
|
|
61
|
+
const projPath = join(CLAUDE_DIR, projDir);
|
|
62
|
+
try { if (!statSync(projPath).isDirectory()) continue; } catch { continue; }
|
|
63
|
+
|
|
64
|
+
const projectName = projDir.replace(/^-/, '/').replace(/-/g, '/').split('/').pop() || projDir;
|
|
65
|
+
const files = readdirSync(projPath).filter(f => f.endsWith('.jsonl') && !f.startsWith('agent-'));
|
|
66
|
+
|
|
67
|
+
for (const file of files) {
|
|
68
|
+
const filePath = join(projPath, file);
|
|
69
|
+
fileCount++;
|
|
70
|
+
let sessionInput = 0, sessionOutput = 0, sessionCost = 0, sessionMsgs = 0;
|
|
71
|
+
|
|
72
|
+
try {
|
|
73
|
+
const content = readFileSync(filePath, 'utf-8');
|
|
74
|
+
for (const line of content.split('\n')) {
|
|
75
|
+
if (!line.trim()) continue;
|
|
76
|
+
try {
|
|
77
|
+
const obj = JSON.parse(line);
|
|
78
|
+
if (obj.type === 'assistant' && obj.message?.usage) {
|
|
79
|
+
const u = obj.message.usage;
|
|
80
|
+
const model = obj.message.model || '';
|
|
81
|
+
const family = getModelFamily(model);
|
|
82
|
+
const input = u.input_tokens || 0;
|
|
83
|
+
const output = u.output_tokens || 0;
|
|
84
|
+
const cacheRead = u.cache_read_input_tokens || 0;
|
|
85
|
+
const cacheCreate = u.cache_creation_input_tokens || 0;
|
|
86
|
+
const cost = calcCost(family, input, output, cacheRead, cacheCreate);
|
|
87
|
+
|
|
88
|
+
sessionInput += input;
|
|
89
|
+
sessionOutput += output;
|
|
90
|
+
sessionCost += cost;
|
|
91
|
+
sessionMsgs++;
|
|
92
|
+
|
|
93
|
+
if (!byModel[family]) byModel[family] = { input: 0, output: 0, cost: 0, messages: 0 };
|
|
94
|
+
byModel[family].input += input;
|
|
95
|
+
byModel[family].output += output;
|
|
96
|
+
byModel[family].cost += cost;
|
|
97
|
+
byModel[family].messages++;
|
|
98
|
+
|
|
99
|
+
const day = (obj.timestamp || '').slice(0, 10) || 'unknown';
|
|
100
|
+
if (!byDay[day]) byDay[day] = { input: 0, output: 0, cost: 0 };
|
|
101
|
+
byDay[day].input += input;
|
|
102
|
+
byDay[day].output += output;
|
|
103
|
+
byDay[day].cost += cost;
|
|
104
|
+
}
|
|
105
|
+
} catch {}
|
|
106
|
+
}
|
|
107
|
+
} catch { continue; }
|
|
108
|
+
|
|
109
|
+
if (sessionMsgs > 0) {
|
|
110
|
+
totalSessions++;
|
|
111
|
+
totalMessages += sessionMsgs;
|
|
112
|
+
totalInput += sessionInput;
|
|
113
|
+
totalOutput += sessionOutput;
|
|
114
|
+
totalCost += sessionCost;
|
|
115
|
+
|
|
116
|
+
if (!byProject[projectName]) byProject[projectName] = { input: 0, output: 0, cost: 0, sessions: 0, messages: 0, cacheRead: 0, cacheCreate: 0 };
|
|
117
|
+
byProject[projectName].input += sessionInput;
|
|
118
|
+
byProject[projectName].output += sessionOutput;
|
|
119
|
+
byProject[projectName].cost += sessionCost;
|
|
120
|
+
byProject[projectName].sessions++;
|
|
121
|
+
byProject[projectName].messages += sessionMsgs;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// Now run the DB scanner and compare
|
|
127
|
+
console.log('Running DB scanner...\n');
|
|
128
|
+
|
|
129
|
+
// Set up environment for the scanner
|
|
130
|
+
process.env.FORGE_DATA_DIR = process.env.FORGE_DATA_DIR || join(homedir(), '.forge', 'data');
|
|
131
|
+
|
|
132
|
+
// Dynamic import to use the actual scanner
|
|
133
|
+
const { scanUsage, queryUsage } = await import('../lib/usage-scanner');
|
|
134
|
+
|
|
135
|
+
const scanResult = scanUsage();
|
|
136
|
+
console.log(`Scan result: ${scanResult.scanned} files scanned, ${scanResult.updated} updated, ${scanResult.errors} errors\n`);
|
|
137
|
+
|
|
138
|
+
const dbData = queryUsage({});
|
|
139
|
+
|
|
140
|
+
// Compare
|
|
141
|
+
console.log('=== COMPARISON ===\n');
|
|
142
|
+
|
|
143
|
+
console.log('TOTAL:');
|
|
144
|
+
console.log(` Direct: ${(totalInput/1000).toFixed(0)}K in, ${(totalOutput/1000).toFixed(0)}K out, $${totalCost.toFixed(2)}, ${totalSessions} sessions, ${totalMessages} msgs`);
|
|
145
|
+
console.log(` DB: ${(dbData.total.input/1000).toFixed(0)}K in, ${(dbData.total.output/1000).toFixed(0)}K out, $${dbData.total.cost.toFixed(2)}, ${dbData.total.sessions} sessions, ${dbData.total.messages} msgs`);
|
|
146
|
+
|
|
147
|
+
const costDiff = Math.abs(totalCost - dbData.total.cost);
|
|
148
|
+
const costMatch = costDiff < 0.1;
|
|
149
|
+
console.log(` Match: ${costMatch ? '✅' : '❌'} (diff: $${costDiff.toFixed(2)})\n`);
|
|
150
|
+
|
|
151
|
+
console.log('BY MODEL:');
|
|
152
|
+
for (const [model, d] of Object.entries(byModel).sort((a, b) => b[1].cost - a[1].cost)) {
|
|
153
|
+
const dbModel = dbData.byModel.find(m => m.model === model);
|
|
154
|
+
const dbCost = dbModel?.cost || 0;
|
|
155
|
+
const match = Math.abs(d.cost - dbCost) < 0.1;
|
|
156
|
+
console.log(` ${model.padEnd(20)} Direct: $${d.cost.toFixed(2).padStart(8)} DB: $${dbCost.toFixed(2).padStart(8)} ${match ? '✅' : '❌'}`);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
console.log('\nBY PROJECT (top 10):');
|
|
160
|
+
const sortedProjects = Object.entries(byProject).sort((a, b) => b[1].cost - a[1].cost).slice(0, 10);
|
|
161
|
+
for (const [name, d] of sortedProjects) {
|
|
162
|
+
const dbProj = dbData.byProject.find(p => p.name === name);
|
|
163
|
+
const dbCost = dbProj?.cost || 0;
|
|
164
|
+
const match = Math.abs(d.cost - dbCost) < 0.1;
|
|
165
|
+
console.log(` ${name.padEnd(25)} Direct: $${d.cost.toFixed(2).padStart(8)} DB: $${dbCost.toFixed(2).padStart(8)} ${match ? '✅' : '❌'}`);
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
console.log('\nBY DAY (last 7):');
|
|
169
|
+
const sortedDays = Object.entries(byDay).filter(([d]) => d !== 'unknown').sort((a, b) => b[0].localeCompare(a[0])).slice(0, 7);
|
|
170
|
+
for (const [day, d] of sortedDays) {
|
|
171
|
+
const dbDay = dbData.byDay.find(dd => dd.date === day);
|
|
172
|
+
const dbCost = dbDay?.cost || 0;
|
|
173
|
+
const match = Math.abs(d.cost - dbCost) < 0.1;
|
|
174
|
+
console.log(` ${day} Direct: $${d.cost.toFixed(2).padStart(8)} DB: $${dbCost.toFixed(2).padStart(8)} ${match ? '✅' : '❌'}`);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
console.log(`\nFiles scanned: ${fileCount}`);
|
|
178
|
+
console.log('');
|
package/src/config/index.ts
CHANGED
package/src/core/db/database.ts
CHANGED
|
@@ -36,6 +36,8 @@ function initSchema(db: Database.Database) {
|
|
|
36
36
|
migrate('ALTER TABLE skills ADD COLUMN deleted_remotely INTEGER NOT NULL DEFAULT 0');
|
|
37
37
|
migrate('ALTER TABLE project_pipelines ADD COLUMN last_run_at TEXT');
|
|
38
38
|
migrate('ALTER TABLE pipeline_runs ADD COLUMN dedup_key TEXT');
|
|
39
|
+
// Recreate token_usage with day column (drop old version if schema changed)
|
|
40
|
+
try { db.exec("SELECT day FROM token_usage LIMIT 1"); } catch { try { db.exec("DROP TABLE IF EXISTS token_usage"); db.exec("DROP TABLE IF EXISTS usage_scan_state"); } catch {} }
|
|
39
41
|
// Unique index for dedup (only applies when dedup_key is NOT NULL)
|
|
40
42
|
try { db.exec('CREATE UNIQUE INDEX IF NOT EXISTS idx_pipeline_runs_dedup ON pipeline_runs(project_path, workflow_name, dedup_key)'); } catch {}
|
|
41
43
|
// Migrate old issue_autofix_processed → pipeline_runs
|
|
@@ -219,6 +221,32 @@ function initSchema(db: Database.Database) {
|
|
|
219
221
|
active INTEGER NOT NULL DEFAULT 1,
|
|
220
222
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
221
223
|
);
|
|
224
|
+
|
|
225
|
+
-- Token usage tracking (per session + day + model for accurate daily breakdown)
|
|
226
|
+
CREATE TABLE IF NOT EXISTS token_usage (
|
|
227
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
228
|
+
session_id TEXT NOT NULL,
|
|
229
|
+
source TEXT NOT NULL DEFAULT 'terminal',
|
|
230
|
+
project_path TEXT NOT NULL,
|
|
231
|
+
project_name TEXT NOT NULL,
|
|
232
|
+
model TEXT NOT NULL DEFAULT 'unknown',
|
|
233
|
+
day TEXT NOT NULL,
|
|
234
|
+
input_tokens INTEGER NOT NULL DEFAULT 0,
|
|
235
|
+
output_tokens INTEGER NOT NULL DEFAULT 0,
|
|
236
|
+
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
|
|
237
|
+
cache_create_tokens INTEGER NOT NULL DEFAULT 0,
|
|
238
|
+
cost_usd REAL NOT NULL DEFAULT 0,
|
|
239
|
+
message_count INTEGER NOT NULL DEFAULT 0,
|
|
240
|
+
task_id TEXT,
|
|
241
|
+
UNIQUE(session_id, source, model, day)
|
|
242
|
+
);
|
|
243
|
+
|
|
244
|
+
-- Track scan progress for incremental JSONL scanning
|
|
245
|
+
CREATE TABLE IF NOT EXISTS usage_scan_state (
|
|
246
|
+
file_path TEXT PRIMARY KEY,
|
|
247
|
+
last_size INTEGER NOT NULL DEFAULT 0,
|
|
248
|
+
last_scan TEXT NOT NULL DEFAULT (datetime('now'))
|
|
249
|
+
);
|
|
222
250
|
`);
|
|
223
251
|
}
|
|
224
252
|
|