@narumitw/pi-analytics 0.49.9 → 0.49.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,260 +2,256 @@ import type { ProviderErrorCategory, SettledRun } from "../types.js";
2
2
 
3
3
  export type TimeRangeId = "today" | "7d" | "30d" | "all";
4
4
  export interface TimeRange {
5
- id?: TimeRangeId;
6
- fromMs: number;
7
- toMs: number;
5
+ id?: TimeRangeId;
6
+ fromMs: number;
7
+ toMs: number;
8
8
  }
9
9
 
10
10
  export interface OverviewStats {
11
- responseCycles: number;
12
- llmCalls: number;
13
- callsPerResponse: number;
14
- p95CallsPerResponse: number;
15
- toolCalls: number;
16
- toolErrors: number;
17
- skillActivations: number;
18
- providerErrors: number;
19
- recoveredErrors: number;
11
+ responseCycles: number;
12
+ llmCalls: number;
13
+ callsPerResponse: number;
14
+ p95CallsPerResponse: number;
15
+ toolCalls: number;
16
+ toolErrors: number;
17
+ skillActivations: number;
18
+ providerErrors: number;
19
+ recoveredErrors: number;
20
20
  }
21
21
 
22
22
  export interface ModelCount {
23
- provider?: string;
24
- model?: string;
25
- count: number;
23
+ provider?: string;
24
+ model?: string;
25
+ count: number;
26
26
  }
27
27
 
28
28
  export interface SkillStats {
29
- name: string;
30
- count: number;
31
- modelInitiated: number;
32
- userInitiated: number;
33
- lastOccurredAtMs: number;
34
- models: ModelCount[];
29
+ name: string;
30
+ count: number;
31
+ modelInitiated: number;
32
+ userInitiated: number;
33
+ lastOccurredAtMs: number;
34
+ models: ModelCount[];
35
35
  }
36
36
 
37
37
  export interface ToolStats {
38
- name: string;
39
- count: number;
40
- errors: number;
41
- averageDurationMs: number;
42
- lastOccurredAtMs: number;
43
- models: ModelCount[];
38
+ name: string;
39
+ count: number;
40
+ errors: number;
41
+ averageDurationMs: number;
42
+ lastOccurredAtMs: number;
43
+ models: ModelCount[];
44
44
  }
45
45
 
46
46
  export interface ReliabilityStats {
47
- http429: number;
48
- http5xx: number;
49
- recovered: number;
50
- terminal: number;
51
- categories: Record<ProviderErrorCategory, number>;
47
+ http429: number;
48
+ http5xx: number;
49
+ recovered: number;
50
+ terminal: number;
51
+ categories: Record<ProviderErrorCategory, number>;
52
52
  }
53
53
 
54
54
  export interface ResponseStats {
55
- count: number;
56
- llmCalls: number;
57
- average: number;
58
- median: number;
59
- p95: number;
60
- maximum: number;
61
- distribution: { one: number; twoToThree: number; fourToSix: number; sevenPlus: number };
55
+ count: number;
56
+ llmCalls: number;
57
+ average: number;
58
+ median: number;
59
+ p95: number;
60
+ maximum: number;
61
+ distribution: { one: number; twoToThree: number; fourToSix: number; sevenPlus: number };
62
62
  }
63
63
 
64
64
  export interface AnalyticsSnapshot {
65
- overview: OverviewStats;
66
- skills: SkillStats[];
67
- tools: ToolStats[];
68
- reliability: ReliabilityStats;
69
- responses: ResponseStats;
65
+ overview: OverviewStats;
66
+ skills: SkillStats[];
67
+ tools: ToolStats[];
68
+ reliability: ReliabilityStats;
69
+ responses: ResponseStats;
70
70
  }
71
71
 
72
72
  const DAY_MS = 24 * 60 * 60 * 1_000;
73
73
  const ERROR_CATEGORIES: readonly ProviderErrorCategory[] = [
74
- "dns",
75
- "timeout",
76
- "connection_refused",
77
- "connection_reset",
78
- "tls",
79
- "network_other",
80
- "provider_other",
74
+ "dns",
75
+ "timeout",
76
+ "connection_refused",
77
+ "connection_reset",
78
+ "tls",
79
+ "network_other",
80
+ "provider_other",
81
81
  ];
82
82
 
83
83
  export function resolveTimeRange(id: TimeRangeId, now = Date.now()): TimeRange {
84
- let fromMs = 0;
85
- if (id === "today") {
86
- const date = new Date(now);
87
- fromMs = new Date(date.getFullYear(), date.getMonth(), date.getDate()).getTime();
88
- } else if (id === "7d") fromMs = now - 7 * DAY_MS;
89
- else if (id === "30d") fromMs = now - 30 * DAY_MS;
90
- return { id, fromMs, toMs: now + 1 };
84
+ let fromMs = 0;
85
+ if (id === "today") {
86
+ const date = new Date(now);
87
+ fromMs = new Date(date.getFullYear(), date.getMonth(), date.getDate()).getTime();
88
+ } else if (id === "7d") fromMs = now - 7 * DAY_MS;
89
+ else if (id === "30d") fromMs = now - 30 * DAY_MS;
90
+ return { id, fromMs, toMs: now + 1 };
91
91
  }
92
92
 
93
93
  export async function querySnapshot(
94
- runs: AsyncIterable<SettledRun> | Iterable<SettledRun>,
95
- range: TimeRange,
96
- signal?: AbortSignal,
94
+ runs: AsyncIterable<SettledRun> | Iterable<SettledRun>,
95
+ range: TimeRange,
96
+ signal?: AbortSignal,
97
97
  ): Promise<AnalyticsSnapshot> {
98
- const generationCounts: number[] = [];
99
- const seenRunIds = new Set<string>();
100
- const skills = new Map<string, SkillStats>();
101
- const tools = new Map<string, ToolStats & { totalDurationMs: number }>();
102
- const categories = Object.fromEntries(
103
- ERROR_CATEGORIES.map((category) => [category, 0]),
104
- ) as Record<ProviderErrorCategory, number>;
105
- let toolErrors = 0;
106
- let providerErrors = 0;
107
- let recoveredErrors = 0;
108
- let http429 = 0;
109
- let http5xx = 0;
110
- let terminal = 0;
98
+ const generationCounts: number[] = [];
99
+ const seenRunIds = new Set<string>();
100
+ const skills = new Map<string, SkillStats>();
101
+ const tools = new Map<string, ToolStats & { totalDurationMs: number }>();
102
+ const categories = Object.fromEntries(ERROR_CATEGORIES.map((category) => [category, 0])) as Record<
103
+ ProviderErrorCategory,
104
+ number
105
+ >;
106
+ let toolErrors = 0;
107
+ let providerErrors = 0;
108
+ let recoveredErrors = 0;
109
+ let http429 = 0;
110
+ let http5xx = 0;
111
+ let terminal = 0;
111
112
 
112
- for await (const run of runs) {
113
- throwIfAborted(signal);
114
- if (seenRunIds.has(run.id)) continue;
115
- seenRunIds.add(run.id);
116
- if (run.startedAtMs < range.fromMs || run.startedAtMs >= range.toMs) continue;
117
- generationCounts.push(run.generations.length);
118
- toolErrors += run.toolErrorCount;
119
- providerErrors += run.providerErrorCount;
120
- recoveredErrors += run.recoveredErrorCount;
113
+ for await (const run of runs) {
114
+ throwIfAborted(signal);
115
+ if (seenRunIds.has(run.id)) continue;
116
+ seenRunIds.add(run.id);
117
+ if (run.startedAtMs < range.fromMs || run.startedAtMs >= range.toMs) continue;
118
+ generationCounts.push(run.generations.length);
119
+ toolErrors += run.toolErrorCount;
120
+ providerErrors += run.providerErrorCount;
121
+ recoveredErrors += run.recoveredErrorCount;
121
122
 
122
- for (const skill of run.skills) {
123
- const item = skills.get(skill.name) ?? {
124
- name: skill.name,
125
- count: 0,
126
- modelInitiated: 0,
127
- userInitiated: 0,
128
- lastOccurredAtMs: 0,
129
- models: [],
130
- };
131
- item.count += 1;
132
- if (skill.initiatedBy === "user") item.userInitiated += 1;
133
- else item.modelInitiated += 1;
134
- item.lastOccurredAtMs = Math.max(item.lastOccurredAtMs, skill.occurredAtMs);
135
- mergeModelCount(item.models, {
136
- provider: skill.provider,
137
- model: skill.model,
138
- count: 1,
139
- });
140
- skills.set(skill.name, item);
141
- }
123
+ for (const skill of run.skills) {
124
+ const item = skills.get(skill.name) ?? {
125
+ name: skill.name,
126
+ count: 0,
127
+ modelInitiated: 0,
128
+ userInitiated: 0,
129
+ lastOccurredAtMs: 0,
130
+ models: [],
131
+ };
132
+ item.count += 1;
133
+ if (skill.initiatedBy === "user") item.userInitiated += 1;
134
+ else item.modelInitiated += 1;
135
+ item.lastOccurredAtMs = Math.max(item.lastOccurredAtMs, skill.occurredAtMs);
136
+ mergeModelCount(item.models, {
137
+ provider: skill.provider,
138
+ model: skill.model,
139
+ count: 1,
140
+ });
141
+ skills.set(skill.name, item);
142
+ }
142
143
 
143
- for (const tool of run.tools) {
144
- const item = tools.get(tool.name) ?? {
145
- name: tool.name,
146
- count: 0,
147
- errors: 0,
148
- averageDurationMs: 0,
149
- totalDurationMs: 0,
150
- lastOccurredAtMs: 0,
151
- models: [],
152
- };
153
- item.count += 1;
154
- item.errors += tool.isError ? 1 : 0;
155
- item.totalDurationMs += tool.durationMs ?? 0;
156
- item.averageDurationMs = item.totalDurationMs / item.count;
157
- item.lastOccurredAtMs = Math.max(item.lastOccurredAtMs, tool.startedAtMs);
158
- mergeModelCount(item.models, {
159
- provider: tool.provider,
160
- model: tool.model,
161
- count: 1,
162
- });
163
- tools.set(tool.name, item);
164
- }
144
+ for (const tool of run.tools) {
145
+ const item = tools.get(tool.name) ?? {
146
+ name: tool.name,
147
+ count: 0,
148
+ errors: 0,
149
+ averageDurationMs: 0,
150
+ totalDurationMs: 0,
151
+ lastOccurredAtMs: 0,
152
+ models: [],
153
+ };
154
+ item.count += 1;
155
+ item.errors += tool.isError ? 1 : 0;
156
+ item.totalDurationMs += tool.durationMs ?? 0;
157
+ item.averageDurationMs = item.totalDurationMs / item.count;
158
+ item.lastOccurredAtMs = Math.max(item.lastOccurredAtMs, tool.startedAtMs);
159
+ mergeModelCount(item.models, {
160
+ provider: tool.provider,
161
+ model: tool.model,
162
+ count: 1,
163
+ });
164
+ tools.set(tool.name, item);
165
+ }
165
166
 
166
- for (const error of run.providerErrors) {
167
- categories[error.category] += 1;
168
- terminal += error.terminal ? 1 : 0;
169
- }
170
- for (const generation of run.generations) {
171
- for (const response of generation.responses) {
172
- if (response.status === 429) http429 += 1;
173
- if (response.status >= 500 && response.status < 600) http5xx += 1;
174
- }
175
- }
176
- }
167
+ for (const error of run.providerErrors) {
168
+ categories[error.category] += 1;
169
+ terminal += error.terminal ? 1 : 0;
170
+ }
171
+ for (const generation of run.generations) {
172
+ for (const response of generation.responses) {
173
+ if (response.status === 429) http429 += 1;
174
+ if (response.status >= 500 && response.status < 600) http5xx += 1;
175
+ }
176
+ }
177
+ }
177
178
 
178
- const responses = responseStatistics(generationCounts);
179
- return {
180
- overview: {
181
- responseCycles: responses.count,
182
- llmCalls: responses.llmCalls,
183
- callsPerResponse: responses.average,
184
- p95CallsPerResponse: responses.p95,
185
- toolCalls: sum([...tools.values()].map(({ count }) => count)),
186
- toolErrors,
187
- skillActivations: sum([...skills.values()].map(({ count }) => count)),
188
- providerErrors,
189
- recoveredErrors,
190
- },
191
- skills: [...skills.values()]
192
- .map((item) => ({ ...item, models: sortModels(item.models) }))
193
- .sort((left, right) => right.count - left.count || left.name.localeCompare(right.name)),
194
- tools: [...tools.values()]
195
- .map(({ totalDurationMs: _, ...item }) => ({ ...item, models: sortModels(item.models) }))
196
- .sort((left, right) => right.count - left.count || left.name.localeCompare(right.name)),
197
- reliability: {
198
- http429,
199
- http5xx,
200
- recovered: recoveredErrors,
201
- terminal,
202
- categories,
203
- },
204
- responses,
205
- };
179
+ const responses = responseStatistics(generationCounts);
180
+ return {
181
+ overview: {
182
+ responseCycles: responses.count,
183
+ llmCalls: responses.llmCalls,
184
+ callsPerResponse: responses.average,
185
+ p95CallsPerResponse: responses.p95,
186
+ toolCalls: sum([...tools.values()].map(({ count }) => count)),
187
+ toolErrors,
188
+ skillActivations: sum([...skills.values()].map(({ count }) => count)),
189
+ providerErrors,
190
+ recoveredErrors,
191
+ },
192
+ skills: [...skills.values()]
193
+ .map((item) => ({ ...item, models: sortModels(item.models) }))
194
+ .sort((left, right) => right.count - left.count || left.name.localeCompare(right.name)),
195
+ tools: [...tools.values()]
196
+ .map(({ totalDurationMs: _, ...item }) => ({ ...item, models: sortModels(item.models) }))
197
+ .sort((left, right) => right.count - left.count || left.name.localeCompare(right.name)),
198
+ reliability: {
199
+ http429,
200
+ http5xx,
201
+ recovered: recoveredErrors,
202
+ terminal,
203
+ categories,
204
+ },
205
+ responses,
206
+ };
206
207
  }
207
208
 
208
209
  function responseStatistics(generationCounts: number[]): ResponseStats {
209
- const sorted = [...generationCounts].sort((left, right) => left - right);
210
- const count = sorted.length;
211
- const llmCalls = sum(sorted);
212
- const nearestRank = (percentile: number) =>
213
- count === 0 ? 0 : (sorted[Math.max(0, Math.ceil(percentile * count) - 1)] ?? 0);
214
- const median =
215
- count === 0
216
- ? 0
217
- : count % 2 === 1
218
- ? (sorted[Math.floor(count / 2)] ?? 0)
219
- : ((sorted[count / 2 - 1] ?? 0) + (sorted[count / 2] ?? 0)) / 2;
220
- return {
221
- count,
222
- llmCalls,
223
- average: count > 0 ? llmCalls / count : 0,
224
- median,
225
- p95: nearestRank(0.95),
226
- maximum: sorted.at(-1) ?? 0,
227
- distribution: {
228
- one: sorted.filter((value) => value === 1).length,
229
- twoToThree: sorted.filter((value) => value >= 2 && value <= 3).length,
230
- fourToSix: sorted.filter((value) => value >= 4 && value <= 6).length,
231
- sevenPlus: sorted.filter((value) => value >= 7).length,
232
- },
233
- };
210
+ const sorted = [...generationCounts].sort((left, right) => left - right);
211
+ const count = sorted.length;
212
+ const llmCalls = sum(sorted);
213
+ const nearestRank = (percentile: number) =>
214
+ count === 0 ? 0 : (sorted[Math.max(0, Math.ceil(percentile * count) - 1)] ?? 0);
215
+ const median =
216
+ count === 0
217
+ ? 0
218
+ : count % 2 === 1
219
+ ? (sorted[Math.floor(count / 2)] ?? 0)
220
+ : ((sorted[count / 2 - 1] ?? 0) + (sorted[count / 2] ?? 0)) / 2;
221
+ return {
222
+ count,
223
+ llmCalls,
224
+ average: count > 0 ? llmCalls / count : 0,
225
+ median,
226
+ p95: nearestRank(0.95),
227
+ maximum: sorted.at(-1) ?? 0,
228
+ distribution: {
229
+ one: sorted.filter((value) => value === 1).length,
230
+ twoToThree: sorted.filter((value) => value >= 2 && value <= 3).length,
231
+ fourToSix: sorted.filter((value) => value >= 4 && value <= 6).length,
232
+ sevenPlus: sorted.filter((value) => value >= 7).length,
233
+ },
234
+ };
234
235
  }
235
236
 
236
237
  function mergeModelCount(models: ModelCount[], next: ModelCount): void {
237
- const existing = models.find(
238
- ({ provider, model }) => provider === next.provider && model === next.model,
239
- );
240
- if (existing) existing.count += next.count;
241
- else models.push(next);
238
+ const existing = models.find(({ provider, model }) => provider === next.provider && model === next.model);
239
+ if (existing) existing.count += next.count;
240
+ else models.push(next);
242
241
  }
243
242
 
244
243
  function sortModels(models: ModelCount[]): ModelCount[] {
245
- return models.sort(
246
- (left, right) =>
247
- right.count - left.count ||
248
- `${left.provider ?? ""}/${left.model ?? ""}`.localeCompare(
249
- `${right.provider ?? ""}/${right.model ?? ""}`,
250
- ),
251
- );
244
+ return models.sort(
245
+ (left, right) =>
246
+ right.count - left.count ||
247
+ `${left.provider ?? ""}/${left.model ?? ""}`.localeCompare(`${right.provider ?? ""}/${right.model ?? ""}`),
248
+ );
252
249
  }
253
250
 
254
251
  function throwIfAborted(signal?: AbortSignal): void {
255
- if (signal?.aborted)
256
- throw signal.reason ?? new DOMException("Analytics query aborted", "AbortError");
252
+ if (signal?.aborted) throw signal.reason ?? new DOMException("Analytics query aborted", "AbortError");
257
253
  }
258
254
 
259
255
  function sum(values: readonly number[]): number {
260
- return values.reduce((total, value) => total + value, 0);
256
+ return values.reduce((total, value) => total + value, 0);
261
257
  }
@@ -1,54 +1,50 @@
1
1
  import type { SettledRun } from "../types.js";
2
- import {
3
- AnalyticsGenerationChangedError,
4
- AnalyticsRunFiles,
5
- type ClearAnalyticsResult,
6
- } from "./files.js";
2
+ import { AnalyticsGenerationChangedError, AnalyticsRunFiles, type ClearAnalyticsResult } from "./files.js";
7
3
  import { type AnalyticsSnapshot, querySnapshot, type TimeRange } from "./queries.js";
8
4
 
9
5
  export class AnalyticsStore {
10
- private readonly files: AnalyticsRunFiles;
6
+ private readonly files: AnalyticsRunFiles;
11
7
 
12
- constructor(
13
- rootPath: string,
14
- dependencies: {
15
- files?: AnalyticsRunFiles;
16
- createId?: () => string;
17
- writeTimeoutMs?: number;
18
- } = {},
19
- ) {
20
- this.files =
21
- dependencies.files ??
22
- new AnalyticsRunFiles(rootPath, {
23
- createId: dependencies.createId,
24
- writeTimeoutMs: dependencies.writeTimeoutMs,
25
- });
26
- }
8
+ constructor(
9
+ rootPath: string,
10
+ dependencies: {
11
+ files?: AnalyticsRunFiles;
12
+ createId?: () => string;
13
+ writeTimeoutMs?: number;
14
+ } = {},
15
+ ) {
16
+ this.files =
17
+ dependencies.files ??
18
+ new AnalyticsRunFiles(rootPath, {
19
+ createId: dependencies.createId,
20
+ writeTimeoutMs: dependencies.writeTimeoutMs,
21
+ });
22
+ }
27
23
 
28
- get path(): string {
29
- return this.files.path;
30
- }
24
+ get path(): string {
25
+ return this.files.path;
26
+ }
31
27
 
32
- recordRun(run: SettledRun, signal?: AbortSignal): Promise<void> {
33
- return this.files.append(run, signal);
34
- }
28
+ recordRun(run: SettledRun, signal?: AbortSignal): Promise<void> {
29
+ return this.files.append(run, signal);
30
+ }
35
31
 
36
- async getSnapshot(range: TimeRange, signal?: AbortSignal): Promise<AnalyticsSnapshot> {
37
- for (let attempt = 0; attempt < 2; attempt += 1) {
38
- try {
39
- return await querySnapshot(this.files.read(signal), range, signal);
40
- } catch (error) {
41
- if (!(error instanceof AnalyticsGenerationChangedError) || attempt > 0) throw error;
42
- }
43
- }
44
- throw new AnalyticsGenerationChangedError();
45
- }
32
+ async getSnapshot(range: TimeRange, signal?: AbortSignal): Promise<AnalyticsSnapshot> {
33
+ for (let attempt = 0; attempt < 2; attempt += 1) {
34
+ try {
35
+ return await querySnapshot(this.files.read(signal), range, signal);
36
+ } catch (error) {
37
+ if (!(error instanceof AnalyticsGenerationChangedError) || attempt > 0) throw error;
38
+ }
39
+ }
40
+ throw new AnalyticsGenerationChangedError();
41
+ }
46
42
 
47
- clearAll(signal?: AbortSignal): Promise<ClearAnalyticsResult> {
48
- return this.files.clear(signal);
49
- }
43
+ clearAll(signal?: AbortSignal): Promise<ClearAnalyticsResult> {
44
+ return this.files.clear(signal);
45
+ }
50
46
 
51
- close(): Promise<void> {
52
- return this.files.close();
53
- }
47
+ close(): Promise<void> {
48
+ return this.files.close();
49
+ }
54
50
  }