jeopi-stats 16.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +195 -0
- package/README.md +82 -0
- package/build.ts +95 -0
- package/dist/client/index.css +1 -0
- package/dist/client/index.html +24 -0
- package/dist/client/index.js +257 -0
- package/dist/client/styles.css +1655 -0
- package/dist/types/aggregator.d.ts +82 -0
- package/dist/types/client/App.d.ts +1 -0
- package/dist/types/client/api.d.ts +20 -0
- package/dist/types/client/app/AppLayout.d.ts +16 -0
- package/dist/types/client/app/NavRail.d.ts +7 -0
- package/dist/types/client/app/RangeControl.d.ts +7 -0
- package/dist/types/client/app/SyncButton.d.ts +14 -0
- package/dist/types/client/app/ThemeToggle.d.ts +1 -0
- package/dist/types/client/app/TopBar.d.ts +15 -0
- package/dist/types/client/app/routes.d.ts +12 -0
- package/dist/types/client/components/AgentTokenShare.d.ts +5 -0
- package/dist/types/client/components/chart-shared.d.ts +173 -0
- package/dist/types/client/components/models-table-shared.d.ts +175 -0
- package/dist/types/client/components/range-meta.d.ts +21 -0
- package/dist/types/client/data/charts.d.ts +1 -0
- package/dist/types/client/data/formatters.d.ts +8 -0
- package/dist/types/client/data/useHashRoute.d.ts +8 -0
- package/dist/types/client/data/useResource.d.ts +13 -0
- package/dist/types/client/data/view-models.d.ts +59 -0
- package/dist/types/client/index.d.ts +2 -0
- package/dist/types/client/routes/BehaviorRoute.d.ts +7 -0
- package/dist/types/client/routes/CostsRoute.d.ts +7 -0
- package/dist/types/client/routes/ErrorsRoute.d.ts +8 -0
- package/dist/types/client/routes/GainRoute.d.ts +7 -0
- package/dist/types/client/routes/ModelsRoute.d.ts +7 -0
- package/dist/types/client/routes/OverviewRoute.d.ts +8 -0
- package/dist/types/client/routes/ProjectsRoute.d.ts +7 -0
- package/dist/types/client/routes/RequestsRoute.d.ts +8 -0
- package/dist/types/client/routes/index.d.ts +8 -0
- package/dist/types/client/types.d.ts +63 -0
- package/dist/types/client/ui/AsyncBoundary.d.ts +12 -0
- package/dist/types/client/ui/DataTable.d.ts +17 -0
- package/dist/types/client/ui/EmptyState.d.ts +7 -0
- package/dist/types/client/ui/ErrorState.d.ts +6 -0
- package/dist/types/client/ui/JsonBlock.d.ts +7 -0
- package/dist/types/client/ui/MetricCluster.d.ts +5 -0
- package/dist/types/client/ui/Panel.d.ts +7 -0
- package/dist/types/client/ui/RequestDrawer.d.ts +5 -0
- package/dist/types/client/ui/SegmentedControl.d.ts +12 -0
- package/dist/types/client/ui/Skeleton.d.ts +8 -0
- package/dist/types/client/ui/StatusPill.d.ts +7 -0
- package/dist/types/client/ui/index.d.ts +11 -0
- package/dist/types/client/useSystemTheme.d.ts +11 -0
- package/dist/types/db.d.ts +114 -0
- package/dist/types/embedded-client.d.ts +18 -0
- package/dist/types/gain-aggregator.d.ts +26 -0
- package/dist/types/index.d.ts +7 -0
- package/dist/types/parser.d.ts +51 -0
- package/dist/types/server.d.ts +7 -0
- package/dist/types/shared-types.d.ts +253 -0
- package/dist/types/sync-worker.d.ts +31 -0
- package/dist/types/types.d.ts +123 -0
- package/dist/types/user-metrics.d.ts +72 -0
- package/package.json +95 -0
- package/src/aggregator.ts +480 -0
- package/src/client/App.tsx +106 -0
- package/src/client/api.ts +94 -0
- package/src/client/app/AppLayout.tsx +93 -0
- package/src/client/app/NavRail.tsx +44 -0
- package/src/client/app/RangeControl.tsx +39 -0
- package/src/client/app/SyncButton.tsx +75 -0
- package/src/client/app/ThemeToggle.tsx +37 -0
- package/src/client/app/TopBar.tsx +73 -0
- package/src/client/app/routes.ts +63 -0
- package/src/client/components/AgentTokenShare.tsx +68 -0
- package/src/client/components/chart-shared.tsx +257 -0
- package/src/client/components/models-table-shared.tsx +255 -0
- package/src/client/components/range-meta.ts +73 -0
- package/src/client/css.d.ts +1 -0
- package/src/client/data/charts.ts +14 -0
- package/src/client/data/formatters.ts +45 -0
- package/src/client/data/useHashRoute.ts +86 -0
- package/src/client/data/useResource.ts +154 -0
- package/src/client/data/view-models.ts +233 -0
- package/src/client/index.tsx +10 -0
- package/src/client/routes/BehaviorRoute.tsx +623 -0
- package/src/client/routes/CostsRoute.tsx +234 -0
- package/src/client/routes/ErrorsRoute.tsx +118 -0
- package/src/client/routes/GainRoute.tsx +226 -0
- package/src/client/routes/ModelsRoute.tsx +430 -0
- package/src/client/routes/OverviewRoute.tsx +342 -0
- package/src/client/routes/ProjectsRoute.tsx +163 -0
- package/src/client/routes/RequestsRoute.tsx +123 -0
- package/src/client/routes/index.ts +8 -0
- package/src/client/styles.css +1333 -0
- package/src/client/types.ts +80 -0
- package/src/client/ui/AsyncBoundary.tsx +54 -0
- package/src/client/ui/DataTable.tsx +122 -0
- package/src/client/ui/EmptyState.tsx +16 -0
- package/src/client/ui/ErrorState.tsx +25 -0
- package/src/client/ui/JsonBlock.tsx +75 -0
- package/src/client/ui/MetricCluster.tsx +67 -0
- package/src/client/ui/Panel.tsx +24 -0
- package/src/client/ui/RequestDrawer.tsx +208 -0
- package/src/client/ui/SegmentedControl.tsx +36 -0
- package/src/client/ui/Skeleton.tsx +17 -0
- package/src/client/ui/StatusPill.tsx +15 -0
- package/src/client/ui/index.ts +11 -0
- package/src/client/useSystemTheme.ts +87 -0
- package/src/db.ts +1278 -0
- package/src/embedded-client.generated.txt +0 -0
- package/src/embedded-client.ts +26 -0
- package/src/gain-aggregator.ts +281 -0
- package/src/index.ts +189 -0
- package/src/parser.ts +368 -0
- package/src/server.ts +346 -0
- package/src/shared-types.ts +271 -0
- package/src/sync-worker.ts +40 -0
- package/src/types.ts +128 -0
- package/src/user-metrics.ts +685 -0
- package/tailwind.config.js +40 -0
package/src/parser.ts
ADDED
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
import * as fs from "node:fs/promises";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import {
|
|
4
|
+
type AssistantMessage,
|
|
5
|
+
coerceServiceTierByFamily,
|
|
6
|
+
getPriorityPremiumRequests,
|
|
7
|
+
resolveModelServiceTier,
|
|
8
|
+
type ServiceTierByFamily,
|
|
9
|
+
} from "jeopi-ai";
|
|
10
|
+
import { getSessionsDir, isEnoent } from "jeopi-utils";
|
|
11
|
+
import type {
|
|
12
|
+
AgentType,
|
|
13
|
+
MessageStats,
|
|
14
|
+
SessionEntry,
|
|
15
|
+
SessionMessageEntry,
|
|
16
|
+
SessionServiceTierChangeEntry,
|
|
17
|
+
UserMessageLink,
|
|
18
|
+
UserMessageStats,
|
|
19
|
+
} from "./types";
|
|
20
|
+
import { computeUserMessageMetrics } from "./user-metrics";
|
|
21
|
+
|
|
22
|
+
/** Basename of an advisor agent's transcript inside a session artifacts dir. */
|
|
23
|
+
const ADVISOR_TRANSCRIPT_BASENAME = "__advisor.jsonl";
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Classify which agent produced a transcript from its path within the sessions
|
|
27
|
+
* directory. Layout: `<sessionsDir>/<project>/<file>.jsonl` is the `main`
|
|
28
|
+
* agent; subagent and advisor transcripts live nested one level deeper inside
|
|
29
|
+
* the session's artifacts dir (`<project>/<session>/<id>.jsonl`,
|
|
30
|
+
* `<project>/<session>/__advisor.jsonl`). Any advisor transcript
|
|
31
|
+
* (`__advisor.jsonl` or `__advisor.<slug>.jsonl`) — at any depth, including a
|
|
32
|
+
* subagent's own advisor — counts as `advisor`; every other nested transcript
|
|
33
|
+
* is a task `subagent`.
|
|
34
|
+
*/
|
|
35
|
+
export function classifyAgentType(sessionPath: string): AgentType {
|
|
36
|
+
const base = path.basename(sessionPath);
|
|
37
|
+
if (base === ADVISOR_TRANSCRIPT_BASENAME || (base.startsWith("__advisor.") && base.endsWith(".jsonl"))) {
|
|
38
|
+
return "advisor";
|
|
39
|
+
}
|
|
40
|
+
const rel = path.relative(getSessionsDir(), sessionPath);
|
|
41
|
+
// `<project>/<file>.jsonl` -> 2 segments. Deeper nesting is a subagent.
|
|
42
|
+
return rel.split(path.sep).length <= 2 ? "main" : "subagent";
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Extract folder name from session filename.
|
|
47
|
+
* Session files are named like: --work--pi--/timestamp_uuid.jsonl
|
|
48
|
+
* The folder part uses -- as path separator.
|
|
49
|
+
*/
|
|
50
|
+
function extractFolderFromPath(sessionPath: string): string {
|
|
51
|
+
const sessionsDir = getSessionsDir();
|
|
52
|
+
const rel = path.relative(sessionsDir, sessionPath);
|
|
53
|
+
const projectDir = rel.split(path.sep)[0];
|
|
54
|
+
// Convert --work--pi-- to /work/pi
|
|
55
|
+
return projectDir.replace(/^--/, "/").replace(/--/g, "/");
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Check if an entry is an assistant message.
|
|
60
|
+
*/
|
|
61
|
+
function isAssistantMessage(entry: SessionEntry): entry is SessionMessageEntry {
|
|
62
|
+
if (entry.type !== "message") return false;
|
|
63
|
+
const msgEntry = entry as SessionMessageEntry;
|
|
64
|
+
// Legacy sessions (pre-id tracking) recorded message entries without an `id`.
|
|
65
|
+
// They're not linkable and would violate the messages.entry_id NOT NULL
|
|
66
|
+
// constraint, so skip them at the parser boundary.
|
|
67
|
+
if (typeof msgEntry.id !== "string" || msgEntry.id.length === 0) return false;
|
|
68
|
+
return msgEntry.message?.role === "assistant";
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Check if an entry is a user message (non-toolResult).
|
|
73
|
+
*/
|
|
74
|
+
function isUserMessage(entry: SessionEntry): entry is SessionMessageEntry {
|
|
75
|
+
if (entry.type !== "message") return false;
|
|
76
|
+
const msgEntry = entry as SessionMessageEntry;
|
|
77
|
+
if (typeof msgEntry.id !== "string" || msgEntry.id.length === 0) return false;
|
|
78
|
+
return msgEntry.message?.role === "user";
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Check if an entry is a service-tier change.
|
|
83
|
+
*/
|
|
84
|
+
function isServiceTierChange(entry: SessionEntry): entry is SessionServiceTierChangeEntry {
|
|
85
|
+
return entry.type === "service_tier_change";
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Extract plain text from a user message content payload.
|
|
90
|
+
*/
|
|
91
|
+
function extractUserText(content: unknown): string {
|
|
92
|
+
if (typeof content === "string") return content;
|
|
93
|
+
if (!Array.isArray(content)) return "";
|
|
94
|
+
const parts: string[] = [];
|
|
95
|
+
for (const block of content) {
|
|
96
|
+
if (block && typeof block === "object" && (block as { type?: unknown }).type === "text") {
|
|
97
|
+
const text = (block as { text?: unknown }).text;
|
|
98
|
+
if (typeof text === "string") parts.push(text);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return parts.join("");
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Build user-message stats from an entry. Returns null for empty/synthetic content.
|
|
106
|
+
*/
|
|
107
|
+
function extractUserStats(sessionFile: string, folder: string, entry: SessionMessageEntry): UserMessageStats | null {
|
|
108
|
+
const msg = entry.message as { role: "user"; content?: unknown; synthetic?: boolean };
|
|
109
|
+
if (msg.role !== "user" || msg.synthetic) return null;
|
|
110
|
+
const text = extractUserText(msg.content);
|
|
111
|
+
if (!text.trim()) return null;
|
|
112
|
+
const metrics = computeUserMessageMetrics(text);
|
|
113
|
+
const ts = Date.parse(entry.timestamp);
|
|
114
|
+
return {
|
|
115
|
+
sessionFile,
|
|
116
|
+
entryId: entry.id,
|
|
117
|
+
folder,
|
|
118
|
+
timestamp: Number.isFinite(ts) ? ts : 0,
|
|
119
|
+
model: null,
|
|
120
|
+
provider: null,
|
|
121
|
+
chars: metrics.chars,
|
|
122
|
+
words: metrics.words,
|
|
123
|
+
yelling: metrics.yelling,
|
|
124
|
+
profanity: metrics.profanity,
|
|
125
|
+
anguish: metrics.anguish,
|
|
126
|
+
negation: metrics.negation,
|
|
127
|
+
repetition: metrics.repetition,
|
|
128
|
+
blame: metrics.blame,
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Extract stats from an assistant message entry.
|
|
134
|
+
*/
|
|
135
|
+
function extractStats(
|
|
136
|
+
sessionFile: string,
|
|
137
|
+
folder: string,
|
|
138
|
+
entry: SessionMessageEntry,
|
|
139
|
+
currentServiceTier: ServiceTierByFamily | undefined,
|
|
140
|
+
agentType: AgentType,
|
|
141
|
+
): MessageStats | null {
|
|
142
|
+
const msg = entry.message as AssistantMessage;
|
|
143
|
+
if (msg?.role !== "assistant") return null;
|
|
144
|
+
|
|
145
|
+
// Backfill: when the session recorded `priority` as the active service tier
|
|
146
|
+
// at this point but the AI usage payload was captured before priority
|
|
147
|
+
// requests were folded into `premiumRequests`, derive the count here so the
|
|
148
|
+
// "Premium Reqs" stat aggregates priority traffic on re-sync. Trust any
|
|
149
|
+
// non-zero value already in `usage.premiumRequests` (Copilot multipliers or
|
|
150
|
+
// the new AI code path) and only synthesise when the field is missing/zero.
|
|
151
|
+
const recorded = msg.usage.premiumRequests ?? 0;
|
|
152
|
+
const model = { provider: msg.provider, api: msg.api, id: msg.model };
|
|
153
|
+
const tier = resolveModelServiceTier(currentServiceTier, model);
|
|
154
|
+
const derived = recorded > 0 ? recorded : getPriorityPremiumRequests(tier, model);
|
|
155
|
+
const usage = derived === recorded ? msg.usage : { ...msg.usage, premiumRequests: derived };
|
|
156
|
+
|
|
157
|
+
return {
|
|
158
|
+
sessionFile,
|
|
159
|
+
entryId: entry.id,
|
|
160
|
+
folder,
|
|
161
|
+
model: msg.model,
|
|
162
|
+
provider: msg.provider,
|
|
163
|
+
api: msg.api,
|
|
164
|
+
timestamp: msg.timestamp,
|
|
165
|
+
duration: msg.duration ?? null,
|
|
166
|
+
ttft: msg.ttft ?? null,
|
|
167
|
+
stopReason: msg.stopReason,
|
|
168
|
+
errorMessage: msg.errorMessage ?? null,
|
|
169
|
+
usage,
|
|
170
|
+
agentType,
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
const LF = 0x0a;
|
|
175
|
+
const CR = 0x0d;
|
|
176
|
+
const jsonLineDecoder = new TextDecoder();
|
|
177
|
+
|
|
178
|
+
function parseJsonLine(bytes: Uint8Array, start: number, end: number): SessionEntry | null {
|
|
179
|
+
while (end > start && bytes[end - 1] === CR) end--;
|
|
180
|
+
if (end <= start) return null;
|
|
181
|
+
try {
|
|
182
|
+
return JSON.parse(jsonLineDecoder.decode(bytes.subarray(start, end))) as SessionEntry;
|
|
183
|
+
} catch {
|
|
184
|
+
return null;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
function visitSessionEntriesLenient(bytes: Uint8Array, visit: (entry: SessionEntry) => void): number {
|
|
189
|
+
let cursor = 0;
|
|
190
|
+
let read = 0;
|
|
191
|
+
|
|
192
|
+
while (cursor < bytes.length) {
|
|
193
|
+
const newline = bytes.indexOf(LF, cursor);
|
|
194
|
+
const hasNewline = newline !== -1;
|
|
195
|
+
const lineEnd = hasNewline ? newline : bytes.length;
|
|
196
|
+
const entry = parseJsonLine(bytes, cursor, lineEnd);
|
|
197
|
+
if (entry) {
|
|
198
|
+
visit(entry);
|
|
199
|
+
read = hasNewline ? newline + 1 : lineEnd;
|
|
200
|
+
} else if (hasNewline) {
|
|
201
|
+
read = newline + 1;
|
|
202
|
+
} else {
|
|
203
|
+
break;
|
|
204
|
+
}
|
|
205
|
+
cursor = hasNewline ? newline + 1 : lineEnd;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
return read;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function parseSessionEntriesLenient(bytes: Uint8Array): { entries: SessionEntry[]; read: number } {
|
|
212
|
+
const entries: SessionEntry[] = [];
|
|
213
|
+
const read = visitSessionEntriesLenient(bytes, entry => entries.push(entry));
|
|
214
|
+
return { entries, read };
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function scanLastServiceTier(bytes: Uint8Array): ServiceTierByFamily | undefined {
|
|
218
|
+
let currentServiceTier: ServiceTierByFamily | undefined;
|
|
219
|
+
visitSessionEntriesLenient(bytes, entry => {
|
|
220
|
+
if (isServiceTierChange(entry)) currentServiceTier = coerceServiceTierByFamily(entry.serviceTier);
|
|
221
|
+
});
|
|
222
|
+
return currentServiceTier;
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Parse a session file and extract all assistant message stats.
|
|
226
|
+
* Uses incremental reading with offset tracking.
|
|
227
|
+
*
|
|
228
|
+
* Service-tier carry-over: `currentServiceTier` is a session-scoped piece of
|
|
229
|
+
* state derived from `service_tier_change` entries that affects whether
|
|
230
|
+
* subsequent OpenAI assistant replies count as premium requests. Incremental
|
|
231
|
+
* syncs that resume past the most-recent tier change would otherwise lose
|
|
232
|
+
* that state and silently record `premiumRequests = 0` for priority traffic
|
|
233
|
+
* (the coding-agent stopped folding the tier into `usage.premiumRequests`
|
|
234
|
+
* after 13f59162e — the parser is now the sole source of truth). When
|
|
235
|
+
* `fromOffset > 0` we therefore scan the bytes preceding `fromOffset`
|
|
236
|
+
* for the latest service-tier value before parsing the unprocessed tail.
|
|
237
|
+
* The scan only keeps the current tier and does not materialize prefix
|
|
238
|
+
* entries, preserving offset-based memory behavior for large sessions.
|
|
239
|
+
*/
|
|
240
|
+
export interface ParseSessionResult {
|
|
241
|
+
stats: MessageStats[];
|
|
242
|
+
userStats: UserMessageStats[];
|
|
243
|
+
userLinks: UserMessageLink[];
|
|
244
|
+
newOffset: number;
|
|
245
|
+
}
|
|
246
|
+
export async function parseSessionFile(sessionPath: string, fromOffset = 0): Promise<ParseSessionResult> {
|
|
247
|
+
let bytes: Uint8Array;
|
|
248
|
+
try {
|
|
249
|
+
bytes = await Bun.file(sessionPath).bytes();
|
|
250
|
+
} catch (err) {
|
|
251
|
+
if (isEnoent(err)) return { stats: [], userStats: [], userLinks: [], newOffset: fromOffset };
|
|
252
|
+
throw err;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
const folder = extractFolderFromPath(sessionPath);
|
|
256
|
+
const agentType = classifyAgentType(sessionPath);
|
|
257
|
+
const stats: MessageStats[] = [];
|
|
258
|
+
const userStats: UserMessageStats[] = [];
|
|
259
|
+
const userLinks: UserMessageLink[] = [];
|
|
260
|
+
const userByEntryId = new Map<string, UserMessageStats>();
|
|
261
|
+
const start = Math.max(0, Math.min(fromOffset, bytes.length));
|
|
262
|
+
const unprocessed = bytes.subarray(start);
|
|
263
|
+
const { entries, read } = parseSessionEntriesLenient(unprocessed);
|
|
264
|
+
let currentServiceTier: ServiceTierByFamily | undefined;
|
|
265
|
+
if (start > 0) {
|
|
266
|
+
currentServiceTier = scanLastServiceTier(bytes.subarray(0, start));
|
|
267
|
+
}
|
|
268
|
+
for (const entry of entries) {
|
|
269
|
+
if (isServiceTierChange(entry)) {
|
|
270
|
+
currentServiceTier = coerceServiceTierByFamily(entry.serviceTier);
|
|
271
|
+
continue;
|
|
272
|
+
}
|
|
273
|
+
if (isUserMessage(entry)) {
|
|
274
|
+
const userMsg = extractUserStats(sessionPath, folder, entry);
|
|
275
|
+
if (userMsg) {
|
|
276
|
+
userStats.push(userMsg);
|
|
277
|
+
userByEntryId.set(entry.id, userMsg);
|
|
278
|
+
}
|
|
279
|
+
continue;
|
|
280
|
+
}
|
|
281
|
+
if (isAssistantMessage(entry)) {
|
|
282
|
+
const msgStats = extractStats(sessionPath, folder, entry, currentServiceTier, agentType);
|
|
283
|
+
if (msgStats) stats.push(msgStats);
|
|
284
|
+
// Link assistant's responding model back to the user message it answered.
|
|
285
|
+
const parentId = (entry as SessionMessageEntry).parentId;
|
|
286
|
+
if (parentId) {
|
|
287
|
+
const msg = entry.message as AssistantMessage;
|
|
288
|
+
if (msg.model && msg.provider) {
|
|
289
|
+
// Emit unconditionally. The aggregator's UPDATE is guarded by
|
|
290
|
+
// `model IS NULL` so this is idempotent: a no-op for already
|
|
291
|
+
// linked rows, a fix-up for fresh inserts (which start NULL
|
|
292
|
+
// because the user row is recorded before its reply lands) and
|
|
293
|
+
// for cross-pass orphans whose parent was committed by an
|
|
294
|
+
// earlier incremental sync.
|
|
295
|
+
userLinks.push({
|
|
296
|
+
sessionFile: sessionPath,
|
|
297
|
+
entryId: parentId,
|
|
298
|
+
model: msg.model,
|
|
299
|
+
provider: msg.provider,
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
return { stats, userStats, userLinks, newOffset: start + read };
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* List all session directories (folders).
|
|
311
|
+
*/
|
|
312
|
+
export async function listSessionFolders(): Promise<string[]> {
|
|
313
|
+
try {
|
|
314
|
+
const sessionsDir = getSessionsDir();
|
|
315
|
+
const entries = await fs.readdir(sessionsDir, { withFileTypes: true });
|
|
316
|
+
return entries.filter(e => e.isDirectory()).map(e => path.join(sessionsDir, e.name));
|
|
317
|
+
} catch {
|
|
318
|
+
return [];
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* List all session files in a folder.
|
|
324
|
+
*/
|
|
325
|
+
export async function listSessionFiles(folderPath: string): Promise<string[]> {
|
|
326
|
+
try {
|
|
327
|
+
const entries = await fs.readdir(folderPath, { recursive: true, withFileTypes: true });
|
|
328
|
+
return entries.filter(e => e.isFile() && e.name.endsWith(".jsonl")).map(e => path.join(e.parentPath, e.name));
|
|
329
|
+
} catch {
|
|
330
|
+
return [];
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
/**
|
|
335
|
+
* List all session files across all folders.
|
|
336
|
+
*/
|
|
337
|
+
export async function listAllSessionFiles(): Promise<string[]> {
|
|
338
|
+
const folders = await listSessionFolders();
|
|
339
|
+
const allFiles: string[] = [];
|
|
340
|
+
|
|
341
|
+
for (const folder of folders) {
|
|
342
|
+
const files = await listSessionFiles(folder);
|
|
343
|
+
allFiles.push(...files);
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
return allFiles;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* Find a specific entry in a session file.
|
|
351
|
+
*/
|
|
352
|
+
export async function getSessionEntry(sessionPath: string, entryId: string): Promise<SessionEntry | null> {
|
|
353
|
+
let bytes: Uint8Array;
|
|
354
|
+
try {
|
|
355
|
+
bytes = await Bun.file(sessionPath).bytes();
|
|
356
|
+
} catch (err) {
|
|
357
|
+
if (isEnoent(err)) return null;
|
|
358
|
+
throw err;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
const { entries } = parseSessionEntriesLenient(bytes);
|
|
362
|
+
for (const entry of entries) {
|
|
363
|
+
if ("id" in entry && entry.id === entryId) {
|
|
364
|
+
return entry;
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
return null;
|
|
368
|
+
}
|
package/src/server.ts
ADDED
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
import type { Dirent } from "node:fs";
|
|
2
|
+
import * as fs from "node:fs/promises";
|
|
3
|
+
import * as os from "node:os";
|
|
4
|
+
import * as path from "node:path";
|
|
5
|
+
import { $ } from "bun";
|
|
6
|
+
import { isEnoent } from "jeopi-utils";
|
|
7
|
+
import {
|
|
8
|
+
getBehaviorDashboardStats,
|
|
9
|
+
getCostDashboardStats,
|
|
10
|
+
getDashboardStats,
|
|
11
|
+
getModelDashboardStats,
|
|
12
|
+
getOverviewStats,
|
|
13
|
+
getRecentErrors,
|
|
14
|
+
getRecentRequests,
|
|
15
|
+
getRequestDetails,
|
|
16
|
+
getTotalMessageCount,
|
|
17
|
+
syncAllSessions,
|
|
18
|
+
} from "./aggregator";
|
|
19
|
+
import { decodeEmbeddedClientArchive } from "./embedded-client";
|
|
20
|
+
import embeddedClientArchiveTxt from "./embedded-client.generated.txt";
|
|
21
|
+
import { getGainDashboardStats } from "./gain-aggregator";
|
|
22
|
+
|
|
23
|
+
const EMBEDDED_CLIENT_ARCHIVE = decodeEmbeddedClientArchive(embeddedClientArchiveTxt);
|
|
24
|
+
|
|
25
|
+
const CLIENT_DIR = path.join(import.meta.dir, "client");
|
|
26
|
+
const STATIC_DIR = path.join(import.meta.dir, "..", "dist", "client");
|
|
27
|
+
const IS_BUN_COMPILED =
|
|
28
|
+
Boolean(process.env.PI_COMPILED || Bun.env.PI_COMPILED) ||
|
|
29
|
+
import.meta.url.includes("$bunfs") ||
|
|
30
|
+
import.meta.url.includes("~BUN") ||
|
|
31
|
+
import.meta.url.includes("%7EBUN");
|
|
32
|
+
// The prepacked npm bundle (coding-agent dist/cli.js) constant-folds
|
|
33
|
+
// process.env.PI_BUNDLED at build time. Like compiled binaries, it ships no
|
|
34
|
+
// dashboard sources or prebuilt dist/client next to the bundle, so the
|
|
35
|
+
// embedded archive is the only viable asset source.
|
|
36
|
+
const IS_PREBUILT = IS_BUN_COMPILED || Boolean(process.env.PI_BUNDLED || Bun.env.PI_BUNDLED);
|
|
37
|
+
const USE_EMBEDDED_CLIENT = EMBEDDED_CLIENT_ARCHIVE !== null || IS_PREBUILT;
|
|
38
|
+
|
|
39
|
+
const EMBEDDED_CLIENT_DIR_ROOT = path.join(os.tmpdir(), "omp-stats-client");
|
|
40
|
+
let embeddedClientDirPromise: Promise<string> | null = null;
|
|
41
|
+
|
|
42
|
+
function sanitizeArchivePath(archivePath: string): string | null {
|
|
43
|
+
const normalized = archivePath.replaceAll("\\", "/").replace(/^\.\//, "");
|
|
44
|
+
if (!normalized || normalized === ".") return null;
|
|
45
|
+
if (normalized.includes("..") || path.isAbsolute(normalized)) return null;
|
|
46
|
+
return normalized;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async function extractEmbeddedClientArchive(archiveBytes: Buffer, outputDir: string): Promise<void> {
|
|
50
|
+
const archive = new Bun.Archive(archiveBytes);
|
|
51
|
+
const files = await archive.files();
|
|
52
|
+
const extractRoot = path.resolve(outputDir);
|
|
53
|
+
|
|
54
|
+
for (const [archivePath, file] of files) {
|
|
55
|
+
const sanitizedPath = sanitizeArchivePath(archivePath);
|
|
56
|
+
if (!sanitizedPath) continue;
|
|
57
|
+
const destinationPath = path.resolve(extractRoot, sanitizedPath);
|
|
58
|
+
if (!destinationPath.startsWith(extractRoot + path.sep)) {
|
|
59
|
+
throw new Error(`Archive entry escapes extraction directory: ${archivePath}`);
|
|
60
|
+
}
|
|
61
|
+
await Bun.write(destinationPath, file);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
async function getEmbeddedClientDir(): Promise<string> {
|
|
66
|
+
if (!USE_EMBEDDED_CLIENT) return STATIC_DIR;
|
|
67
|
+
if (embeddedClientDirPromise) return embeddedClientDirPromise;
|
|
68
|
+
|
|
69
|
+
if (!EMBEDDED_CLIENT_ARCHIVE) {
|
|
70
|
+
throw new Error(
|
|
71
|
+
"Embedded stats client bundle missing. Rebuild the omp binary or npm bundle with embedded stats assets.",
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
embeddedClientDirPromise = (async () => {
|
|
76
|
+
const bundleHash = Bun.hash(EMBEDDED_CLIENT_ARCHIVE).toString(16);
|
|
77
|
+
const outputDir = path.join(EMBEDDED_CLIENT_DIR_ROOT, bundleHash);
|
|
78
|
+
const markerPath = path.join(outputDir, "index.html");
|
|
79
|
+
try {
|
|
80
|
+
const marker = await fs.stat(markerPath);
|
|
81
|
+
if (marker.isFile()) return outputDir;
|
|
82
|
+
} catch {}
|
|
83
|
+
|
|
84
|
+
await fs.rm(outputDir, { recursive: true, force: true });
|
|
85
|
+
await fs.mkdir(outputDir, { recursive: true });
|
|
86
|
+
await extractEmbeddedClientArchive(EMBEDDED_CLIENT_ARCHIVE, outputDir);
|
|
87
|
+
return outputDir;
|
|
88
|
+
})();
|
|
89
|
+
|
|
90
|
+
return embeddedClientDirPromise;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async function getLatestMtime(dir: string): Promise<number> {
|
|
94
|
+
let entries: Dirent[];
|
|
95
|
+
try {
|
|
96
|
+
entries = await fs.readdir(dir, { withFileTypes: true });
|
|
97
|
+
} catch (err) {
|
|
98
|
+
// Tolerate missing source trees (e.g. installs without the dashboard
|
|
99
|
+
// sources); the caller falls back to prebuilt assets or a clear build
|
|
100
|
+
// failure instead of crashing on the scan.
|
|
101
|
+
if (isEnoent(err)) return 0;
|
|
102
|
+
throw err;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const promises = [];
|
|
106
|
+
for (const entry of entries) {
|
|
107
|
+
const fullPath = path.join(dir, entry.name);
|
|
108
|
+
if (entry.isDirectory()) {
|
|
109
|
+
promises.push(getLatestMtime(fullPath));
|
|
110
|
+
} else if (entry.isFile()) {
|
|
111
|
+
promises.push(fs.stat(fullPath).then(stats => stats.mtimeMs));
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
let latest = 0;
|
|
116
|
+
await Promise.allSettled(promises).then(results => {
|
|
117
|
+
for (const result of results) {
|
|
118
|
+
if (result.status === "fulfilled") {
|
|
119
|
+
latest = Math.max(latest, result.value);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
});
|
|
123
|
+
return latest;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const ensureClientBuild = async () => {
|
|
127
|
+
if (USE_EMBEDDED_CLIENT) return;
|
|
128
|
+
const indexPath = path.join(STATIC_DIR, "index.html");
|
|
129
|
+
const cssPath = path.join(STATIC_DIR, "styles.css");
|
|
130
|
+
const clientSourceMtime = await getLatestMtime(CLIENT_DIR);
|
|
131
|
+
const tailwindConfigPath = path.join(import.meta.dir, "..", "tailwind.config.js");
|
|
132
|
+
let tailwindConfigMtime = 0;
|
|
133
|
+
try {
|
|
134
|
+
const tailwindConfigStats = await fs.stat(tailwindConfigPath);
|
|
135
|
+
tailwindConfigMtime = tailwindConfigStats.mtimeMs;
|
|
136
|
+
} catch {}
|
|
137
|
+
const sourceMtime = Math.max(clientSourceMtime, tailwindConfigMtime);
|
|
138
|
+
let shouldBuild = true;
|
|
139
|
+
try {
|
|
140
|
+
const [indexStats, cssStats] = await Promise.all([fs.stat(indexPath), fs.stat(cssPath)]);
|
|
141
|
+
if (
|
|
142
|
+
indexStats.isFile() &&
|
|
143
|
+
cssStats.isFile() &&
|
|
144
|
+
indexStats.mtimeMs >= sourceMtime &&
|
|
145
|
+
cssStats.mtimeMs >= sourceMtime
|
|
146
|
+
) {
|
|
147
|
+
shouldBuild = false;
|
|
148
|
+
}
|
|
149
|
+
} catch {
|
|
150
|
+
shouldBuild = true;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
if (!shouldBuild) return;
|
|
154
|
+
|
|
155
|
+
await fs.rm(STATIC_DIR, { recursive: true, force: true });
|
|
156
|
+
|
|
157
|
+
console.log("Building stats client...");
|
|
158
|
+
const packageRoot = path.join(import.meta.dir, "..");
|
|
159
|
+
const buildResult = await $`bun run build.ts`.cwd(packageRoot).quiet().nothrow();
|
|
160
|
+
if (buildResult.exitCode !== 0) {
|
|
161
|
+
const output = buildResult.text().trim();
|
|
162
|
+
const details = output ? `\n${output}` : "";
|
|
163
|
+
throw new Error(`Failed to build stats client (exit ${buildResult.exitCode})${details}`);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const indexHtml = `<!DOCTYPE html>
|
|
167
|
+
<html lang="en">
|
|
168
|
+
<head>
|
|
169
|
+
<meta charset="UTF-8">
|
|
170
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
171
|
+
<title>jeopi · stats</title>
|
|
172
|
+
<link rel="stylesheet" href="styles.css">
|
|
173
|
+
</head>
|
|
174
|
+
<body>
|
|
175
|
+
<div id="root"></div>
|
|
176
|
+
<script src="index.js" type="module"></script>
|
|
177
|
+
</body>
|
|
178
|
+
</html>`;
|
|
179
|
+
|
|
180
|
+
await Bun.write(path.join(STATIC_DIR, "index.html"), indexHtml);
|
|
181
|
+
};
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* Handle API requests.
|
|
185
|
+
*/
|
|
186
|
+
async function handleApi(req: Request): Promise<Response> {
|
|
187
|
+
const url = new URL(req.url);
|
|
188
|
+
const path = url.pathname;
|
|
189
|
+
|
|
190
|
+
// Stats reads are DB-only; explicit /api/sync does the expensive session scan.
|
|
191
|
+
const range = url.searchParams.get("range");
|
|
192
|
+
|
|
193
|
+
if (path === "/api/stats") {
|
|
194
|
+
const stats = await getDashboardStats(range);
|
|
195
|
+
return Response.json(stats);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
if (path === "/api/stats/overview") {
|
|
199
|
+
const stats = await getOverviewStats(range);
|
|
200
|
+
return Response.json(stats);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
if (path === "/api/stats/model-dashboard") {
|
|
204
|
+
const stats = await getModelDashboardStats(range);
|
|
205
|
+
return Response.json(stats);
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
if (path === "/api/stats/costs") {
|
|
209
|
+
const stats = await getCostDashboardStats(range);
|
|
210
|
+
return Response.json(stats);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
if (path === "/api/stats/behavior") {
|
|
214
|
+
const stats = await getBehaviorDashboardStats(range);
|
|
215
|
+
return Response.json(stats);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
if (path === "/api/stats/recent") {
|
|
219
|
+
const limit = url.searchParams.get("limit");
|
|
220
|
+
const stats = await getRecentRequests(limit ? parseInt(limit, 10) : undefined);
|
|
221
|
+
return Response.json(stats);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
if (path === "/api/stats/errors") {
|
|
225
|
+
const limit = url.searchParams.get("limit");
|
|
226
|
+
const stats = await getRecentErrors(limit ? parseInt(limit, 10) : undefined);
|
|
227
|
+
return Response.json(stats);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
if (path === "/api/stats/models") {
|
|
231
|
+
const stats = await getDashboardStats(range);
|
|
232
|
+
return Response.json(stats.byModel);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
if (path === "/api/stats/folders") {
|
|
236
|
+
const stats = await getDashboardStats(range);
|
|
237
|
+
return Response.json(stats.byFolder);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
if (path === "/api/stats/timeseries") {
|
|
241
|
+
const stats = await getDashboardStats(range);
|
|
242
|
+
return Response.json(stats.timeSeries);
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
if (path.startsWith("/api/request/")) {
|
|
246
|
+
const id = path.split("/").pop();
|
|
247
|
+
if (!id) return new Response("Bad Request", { status: 400 });
|
|
248
|
+
const details = await getRequestDetails(parseInt(id, 10));
|
|
249
|
+
if (!details) return new Response("Not Found", { status: 404 });
|
|
250
|
+
return Response.json(details);
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
if (path === "/api/sync") {
|
|
254
|
+
const result = await syncAllSessions();
|
|
255
|
+
const count = await getTotalMessageCount();
|
|
256
|
+
return Response.json({ ...result, totalMessages: count });
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
if (path === "/api/stats/gain") {
|
|
260
|
+
const project = url.searchParams.get("project");
|
|
261
|
+
const stats = await getGainDashboardStats(range, project);
|
|
262
|
+
return Response.json(stats);
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
return new Response("Not Found", { status: 404 });
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* Handle static file requests.
|
|
270
|
+
*/
|
|
271
|
+
async function handleStatic(requestPath: string): Promise<Response> {
|
|
272
|
+
const staticDir = await getEmbeddedClientDir();
|
|
273
|
+
const filePath = requestPath === "/" ? "/index.html" : requestPath;
|
|
274
|
+
const fullPath = path.join(staticDir, filePath);
|
|
275
|
+
|
|
276
|
+
const file = Bun.file(fullPath);
|
|
277
|
+
if (await file.exists()) {
|
|
278
|
+
return new Response(file);
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// SPA fallback
|
|
282
|
+
const index = Bun.file(path.join(staticDir, "index.html"));
|
|
283
|
+
if (await index.exists()) {
|
|
284
|
+
return new Response(index);
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
return new Response("Not Found", { status: 404 });
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/**
|
|
291
|
+
* Start the HTTP server.
|
|
292
|
+
*/
|
|
293
|
+
export async function startServer(port = 3847): Promise<{ port: number; stop: () => void }> {
|
|
294
|
+
await ensureClientBuild();
|
|
295
|
+
|
|
296
|
+
const server = Bun.serve({
|
|
297
|
+
port,
|
|
298
|
+
async fetch(req) {
|
|
299
|
+
const url = new URL(req.url);
|
|
300
|
+
const path = url.pathname;
|
|
301
|
+
|
|
302
|
+
// CORS headers for local development
|
|
303
|
+
const corsHeaders = {
|
|
304
|
+
"Access-Control-Allow-Origin": "*",
|
|
305
|
+
"Access-Control-Allow-Methods": "GET, POST, OPTIONS",
|
|
306
|
+
"Access-Control-Allow-Headers": "Content-Type",
|
|
307
|
+
};
|
|
308
|
+
|
|
309
|
+
if (req.method === "OPTIONS") {
|
|
310
|
+
return new Response(null, { headers: corsHeaders });
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
try {
|
|
314
|
+
let response: Response;
|
|
315
|
+
|
|
316
|
+
if (path.startsWith("/api/")) {
|
|
317
|
+
response = await handleApi(req);
|
|
318
|
+
} else {
|
|
319
|
+
response = await handleStatic(path);
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// Add CORS headers to all responses
|
|
323
|
+
const headers = new Headers(response.headers);
|
|
324
|
+
for (const [key, value] of Object.entries(corsHeaders)) {
|
|
325
|
+
headers.set(key, value);
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
return new Response(response.body, {
|
|
329
|
+
status: response.status,
|
|
330
|
+
headers,
|
|
331
|
+
});
|
|
332
|
+
} catch (error) {
|
|
333
|
+
console.error("Server error:", error);
|
|
334
|
+
return Response.json(
|
|
335
|
+
{ error: error instanceof Error ? error.message : "Unknown error" },
|
|
336
|
+
{ status: 500, headers: corsHeaders },
|
|
337
|
+
);
|
|
338
|
+
}
|
|
339
|
+
},
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
return {
|
|
343
|
+
port: server.port ?? port,
|
|
344
|
+
stop: () => server.stop(),
|
|
345
|
+
};
|
|
346
|
+
}
|