@narumitw/pi-analytics 0.49.9 → 0.49.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -35
- package/dist/index.ts +53 -106
- package/dist/index.ts.map +2 -2
- package/package.json +49 -50
- package/src/analytics.ts +368 -358
- package/src/collector.ts +250 -262
- package/src/errors.ts +12 -12
- package/src/menu.ts +325 -343
- package/src/skills.ts +55 -57
- package/src/storage/files.ts +352 -374
- package/src/storage/format.ts +159 -201
- package/src/storage/queries.ts +200 -204
- package/src/storage/store.ts +39 -43
- package/src/types.ts +66 -79
package/src/storage/queries.ts
CHANGED
|
@@ -2,260 +2,256 @@ import type { ProviderErrorCategory, SettledRun } from "../types.js";
|
|
|
2
2
|
|
|
3
3
|
export type TimeRangeId = "today" | "7d" | "30d" | "all";
|
|
4
4
|
export interface TimeRange {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
id?: TimeRangeId;
|
|
6
|
+
fromMs: number;
|
|
7
|
+
toMs: number;
|
|
8
8
|
}
|
|
9
9
|
|
|
10
10
|
export interface OverviewStats {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
11
|
+
responseCycles: number;
|
|
12
|
+
llmCalls: number;
|
|
13
|
+
callsPerResponse: number;
|
|
14
|
+
p95CallsPerResponse: number;
|
|
15
|
+
toolCalls: number;
|
|
16
|
+
toolErrors: number;
|
|
17
|
+
skillActivations: number;
|
|
18
|
+
providerErrors: number;
|
|
19
|
+
recoveredErrors: number;
|
|
20
20
|
}
|
|
21
21
|
|
|
22
22
|
export interface ModelCount {
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
23
|
+
provider?: string;
|
|
24
|
+
model?: string;
|
|
25
|
+
count: number;
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
export interface SkillStats {
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
29
|
+
name: string;
|
|
30
|
+
count: number;
|
|
31
|
+
modelInitiated: number;
|
|
32
|
+
userInitiated: number;
|
|
33
|
+
lastOccurredAtMs: number;
|
|
34
|
+
models: ModelCount[];
|
|
35
35
|
}
|
|
36
36
|
|
|
37
37
|
export interface ToolStats {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
38
|
+
name: string;
|
|
39
|
+
count: number;
|
|
40
|
+
errors: number;
|
|
41
|
+
averageDurationMs: number;
|
|
42
|
+
lastOccurredAtMs: number;
|
|
43
|
+
models: ModelCount[];
|
|
44
44
|
}
|
|
45
45
|
|
|
46
46
|
export interface ReliabilityStats {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
47
|
+
http429: number;
|
|
48
|
+
http5xx: number;
|
|
49
|
+
recovered: number;
|
|
50
|
+
terminal: number;
|
|
51
|
+
categories: Record<ProviderErrorCategory, number>;
|
|
52
52
|
}
|
|
53
53
|
|
|
54
54
|
export interface ResponseStats {
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
55
|
+
count: number;
|
|
56
|
+
llmCalls: number;
|
|
57
|
+
average: number;
|
|
58
|
+
median: number;
|
|
59
|
+
p95: number;
|
|
60
|
+
maximum: number;
|
|
61
|
+
distribution: { one: number; twoToThree: number; fourToSix: number; sevenPlus: number };
|
|
62
62
|
}
|
|
63
63
|
|
|
64
64
|
export interface AnalyticsSnapshot {
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
65
|
+
overview: OverviewStats;
|
|
66
|
+
skills: SkillStats[];
|
|
67
|
+
tools: ToolStats[];
|
|
68
|
+
reliability: ReliabilityStats;
|
|
69
|
+
responses: ResponseStats;
|
|
70
70
|
}
|
|
71
71
|
|
|
72
72
|
const DAY_MS = 24 * 60 * 60 * 1_000;
|
|
73
73
|
const ERROR_CATEGORIES: readonly ProviderErrorCategory[] = [
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
74
|
+
"dns",
|
|
75
|
+
"timeout",
|
|
76
|
+
"connection_refused",
|
|
77
|
+
"connection_reset",
|
|
78
|
+
"tls",
|
|
79
|
+
"network_other",
|
|
80
|
+
"provider_other",
|
|
81
81
|
];
|
|
82
82
|
|
|
83
83
|
export function resolveTimeRange(id: TimeRangeId, now = Date.now()): TimeRange {
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
84
|
+
let fromMs = 0;
|
|
85
|
+
if (id === "today") {
|
|
86
|
+
const date = new Date(now);
|
|
87
|
+
fromMs = new Date(date.getFullYear(), date.getMonth(), date.getDate()).getTime();
|
|
88
|
+
} else if (id === "7d") fromMs = now - 7 * DAY_MS;
|
|
89
|
+
else if (id === "30d") fromMs = now - 30 * DAY_MS;
|
|
90
|
+
return { id, fromMs, toMs: now + 1 };
|
|
91
91
|
}
|
|
92
92
|
|
|
93
93
|
export async function querySnapshot(
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
94
|
+
runs: AsyncIterable<SettledRun> | Iterable<SettledRun>,
|
|
95
|
+
range: TimeRange,
|
|
96
|
+
signal?: AbortSignal,
|
|
97
97
|
): Promise<AnalyticsSnapshot> {
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
98
|
+
const generationCounts: number[] = [];
|
|
99
|
+
const seenRunIds = new Set<string>();
|
|
100
|
+
const skills = new Map<string, SkillStats>();
|
|
101
|
+
const tools = new Map<string, ToolStats & { totalDurationMs: number }>();
|
|
102
|
+
const categories = Object.fromEntries(ERROR_CATEGORIES.map((category) => [category, 0])) as Record<
|
|
103
|
+
ProviderErrorCategory,
|
|
104
|
+
number
|
|
105
|
+
>;
|
|
106
|
+
let toolErrors = 0;
|
|
107
|
+
let providerErrors = 0;
|
|
108
|
+
let recoveredErrors = 0;
|
|
109
|
+
let http429 = 0;
|
|
110
|
+
let http5xx = 0;
|
|
111
|
+
let terminal = 0;
|
|
111
112
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
113
|
+
for await (const run of runs) {
|
|
114
|
+
throwIfAborted(signal);
|
|
115
|
+
if (seenRunIds.has(run.id)) continue;
|
|
116
|
+
seenRunIds.add(run.id);
|
|
117
|
+
if (run.startedAtMs < range.fromMs || run.startedAtMs >= range.toMs) continue;
|
|
118
|
+
generationCounts.push(run.generations.length);
|
|
119
|
+
toolErrors += run.toolErrorCount;
|
|
120
|
+
providerErrors += run.providerErrorCount;
|
|
121
|
+
recoveredErrors += run.recoveredErrorCount;
|
|
121
122
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
123
|
+
for (const skill of run.skills) {
|
|
124
|
+
const item = skills.get(skill.name) ?? {
|
|
125
|
+
name: skill.name,
|
|
126
|
+
count: 0,
|
|
127
|
+
modelInitiated: 0,
|
|
128
|
+
userInitiated: 0,
|
|
129
|
+
lastOccurredAtMs: 0,
|
|
130
|
+
models: [],
|
|
131
|
+
};
|
|
132
|
+
item.count += 1;
|
|
133
|
+
if (skill.initiatedBy === "user") item.userInitiated += 1;
|
|
134
|
+
else item.modelInitiated += 1;
|
|
135
|
+
item.lastOccurredAtMs = Math.max(item.lastOccurredAtMs, skill.occurredAtMs);
|
|
136
|
+
mergeModelCount(item.models, {
|
|
137
|
+
provider: skill.provider,
|
|
138
|
+
model: skill.model,
|
|
139
|
+
count: 1,
|
|
140
|
+
});
|
|
141
|
+
skills.set(skill.name, item);
|
|
142
|
+
}
|
|
142
143
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
144
|
+
for (const tool of run.tools) {
|
|
145
|
+
const item = tools.get(tool.name) ?? {
|
|
146
|
+
name: tool.name,
|
|
147
|
+
count: 0,
|
|
148
|
+
errors: 0,
|
|
149
|
+
averageDurationMs: 0,
|
|
150
|
+
totalDurationMs: 0,
|
|
151
|
+
lastOccurredAtMs: 0,
|
|
152
|
+
models: [],
|
|
153
|
+
};
|
|
154
|
+
item.count += 1;
|
|
155
|
+
item.errors += tool.isError ? 1 : 0;
|
|
156
|
+
item.totalDurationMs += tool.durationMs ?? 0;
|
|
157
|
+
item.averageDurationMs = item.totalDurationMs / item.count;
|
|
158
|
+
item.lastOccurredAtMs = Math.max(item.lastOccurredAtMs, tool.startedAtMs);
|
|
159
|
+
mergeModelCount(item.models, {
|
|
160
|
+
provider: tool.provider,
|
|
161
|
+
model: tool.model,
|
|
162
|
+
count: 1,
|
|
163
|
+
});
|
|
164
|
+
tools.set(tool.name, item);
|
|
165
|
+
}
|
|
165
166
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
167
|
+
for (const error of run.providerErrors) {
|
|
168
|
+
categories[error.category] += 1;
|
|
169
|
+
terminal += error.terminal ? 1 : 0;
|
|
170
|
+
}
|
|
171
|
+
for (const generation of run.generations) {
|
|
172
|
+
for (const response of generation.responses) {
|
|
173
|
+
if (response.status === 429) http429 += 1;
|
|
174
|
+
if (response.status >= 500 && response.status < 600) http5xx += 1;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
177
178
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
179
|
+
const responses = responseStatistics(generationCounts);
|
|
180
|
+
return {
|
|
181
|
+
overview: {
|
|
182
|
+
responseCycles: responses.count,
|
|
183
|
+
llmCalls: responses.llmCalls,
|
|
184
|
+
callsPerResponse: responses.average,
|
|
185
|
+
p95CallsPerResponse: responses.p95,
|
|
186
|
+
toolCalls: sum([...tools.values()].map(({ count }) => count)),
|
|
187
|
+
toolErrors,
|
|
188
|
+
skillActivations: sum([...skills.values()].map(({ count }) => count)),
|
|
189
|
+
providerErrors,
|
|
190
|
+
recoveredErrors,
|
|
191
|
+
},
|
|
192
|
+
skills: [...skills.values()]
|
|
193
|
+
.map((item) => ({ ...item, models: sortModels(item.models) }))
|
|
194
|
+
.sort((left, right) => right.count - left.count || left.name.localeCompare(right.name)),
|
|
195
|
+
tools: [...tools.values()]
|
|
196
|
+
.map(({ totalDurationMs: _, ...item }) => ({ ...item, models: sortModels(item.models) }))
|
|
197
|
+
.sort((left, right) => right.count - left.count || left.name.localeCompare(right.name)),
|
|
198
|
+
reliability: {
|
|
199
|
+
http429,
|
|
200
|
+
http5xx,
|
|
201
|
+
recovered: recoveredErrors,
|
|
202
|
+
terminal,
|
|
203
|
+
categories,
|
|
204
|
+
},
|
|
205
|
+
responses,
|
|
206
|
+
};
|
|
206
207
|
}
|
|
207
208
|
|
|
208
209
|
function responseStatistics(generationCounts: number[]): ResponseStats {
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
210
|
+
const sorted = [...generationCounts].sort((left, right) => left - right);
|
|
211
|
+
const count = sorted.length;
|
|
212
|
+
const llmCalls = sum(sorted);
|
|
213
|
+
const nearestRank = (percentile: number) =>
|
|
214
|
+
count === 0 ? 0 : (sorted[Math.max(0, Math.ceil(percentile * count) - 1)] ?? 0);
|
|
215
|
+
const median =
|
|
216
|
+
count === 0
|
|
217
|
+
? 0
|
|
218
|
+
: count % 2 === 1
|
|
219
|
+
? (sorted[Math.floor(count / 2)] ?? 0)
|
|
220
|
+
: ((sorted[count / 2 - 1] ?? 0) + (sorted[count / 2] ?? 0)) / 2;
|
|
221
|
+
return {
|
|
222
|
+
count,
|
|
223
|
+
llmCalls,
|
|
224
|
+
average: count > 0 ? llmCalls / count : 0,
|
|
225
|
+
median,
|
|
226
|
+
p95: nearestRank(0.95),
|
|
227
|
+
maximum: sorted.at(-1) ?? 0,
|
|
228
|
+
distribution: {
|
|
229
|
+
one: sorted.filter((value) => value === 1).length,
|
|
230
|
+
twoToThree: sorted.filter((value) => value >= 2 && value <= 3).length,
|
|
231
|
+
fourToSix: sorted.filter((value) => value >= 4 && value <= 6).length,
|
|
232
|
+
sevenPlus: sorted.filter((value) => value >= 7).length,
|
|
233
|
+
},
|
|
234
|
+
};
|
|
234
235
|
}
|
|
235
236
|
|
|
236
237
|
function mergeModelCount(models: ModelCount[], next: ModelCount): void {
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
if (existing) existing.count += next.count;
|
|
241
|
-
else models.push(next);
|
|
238
|
+
const existing = models.find(({ provider, model }) => provider === next.provider && model === next.model);
|
|
239
|
+
if (existing) existing.count += next.count;
|
|
240
|
+
else models.push(next);
|
|
242
241
|
}
|
|
243
242
|
|
|
244
243
|
function sortModels(models: ModelCount[]): ModelCount[] {
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
),
|
|
251
|
-
);
|
|
244
|
+
return models.sort(
|
|
245
|
+
(left, right) =>
|
|
246
|
+
right.count - left.count ||
|
|
247
|
+
`${left.provider ?? ""}/${left.model ?? ""}`.localeCompare(`${right.provider ?? ""}/${right.model ?? ""}`),
|
|
248
|
+
);
|
|
252
249
|
}
|
|
253
250
|
|
|
254
251
|
function throwIfAborted(signal?: AbortSignal): void {
|
|
255
|
-
|
|
256
|
-
throw signal.reason ?? new DOMException("Analytics query aborted", "AbortError");
|
|
252
|
+
if (signal?.aborted) throw signal.reason ?? new DOMException("Analytics query aborted", "AbortError");
|
|
257
253
|
}
|
|
258
254
|
|
|
259
255
|
function sum(values: readonly number[]): number {
|
|
260
|
-
|
|
256
|
+
return values.reduce((total, value) => total + value, 0);
|
|
261
257
|
}
|
package/src/storage/store.ts
CHANGED
|
@@ -1,54 +1,50 @@
|
|
|
1
1
|
import type { SettledRun } from "../types.js";
|
|
2
|
-
import {
|
|
3
|
-
AnalyticsGenerationChangedError,
|
|
4
|
-
AnalyticsRunFiles,
|
|
5
|
-
type ClearAnalyticsResult,
|
|
6
|
-
} from "./files.js";
|
|
2
|
+
import { AnalyticsGenerationChangedError, AnalyticsRunFiles, type ClearAnalyticsResult } from "./files.js";
|
|
7
3
|
import { type AnalyticsSnapshot, querySnapshot, type TimeRange } from "./queries.js";
|
|
8
4
|
|
|
9
5
|
export class AnalyticsStore {
|
|
10
|
-
|
|
6
|
+
private readonly files: AnalyticsRunFiles;
|
|
11
7
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
8
|
+
constructor(
|
|
9
|
+
rootPath: string,
|
|
10
|
+
dependencies: {
|
|
11
|
+
files?: AnalyticsRunFiles;
|
|
12
|
+
createId?: () => string;
|
|
13
|
+
writeTimeoutMs?: number;
|
|
14
|
+
} = {},
|
|
15
|
+
) {
|
|
16
|
+
this.files =
|
|
17
|
+
dependencies.files ??
|
|
18
|
+
new AnalyticsRunFiles(rootPath, {
|
|
19
|
+
createId: dependencies.createId,
|
|
20
|
+
writeTimeoutMs: dependencies.writeTimeoutMs,
|
|
21
|
+
});
|
|
22
|
+
}
|
|
27
23
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
24
|
+
get path(): string {
|
|
25
|
+
return this.files.path;
|
|
26
|
+
}
|
|
31
27
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
28
|
+
recordRun(run: SettledRun, signal?: AbortSignal): Promise<void> {
|
|
29
|
+
return this.files.append(run, signal);
|
|
30
|
+
}
|
|
35
31
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
32
|
+
async getSnapshot(range: TimeRange, signal?: AbortSignal): Promise<AnalyticsSnapshot> {
|
|
33
|
+
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
34
|
+
try {
|
|
35
|
+
return await querySnapshot(this.files.read(signal), range, signal);
|
|
36
|
+
} catch (error) {
|
|
37
|
+
if (!(error instanceof AnalyticsGenerationChangedError) || attempt > 0) throw error;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
throw new AnalyticsGenerationChangedError();
|
|
41
|
+
}
|
|
46
42
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
43
|
+
clearAll(signal?: AbortSignal): Promise<ClearAnalyticsResult> {
|
|
44
|
+
return this.files.clear(signal);
|
|
45
|
+
}
|
|
50
46
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
47
|
+
close(): Promise<void> {
|
|
48
|
+
return this.files.close();
|
|
49
|
+
}
|
|
54
50
|
}
|