@narumitw/pi-analytics 0.49.9 → 0.49.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/collector.ts CHANGED
@@ -1,301 +1,289 @@
1
1
  import { randomUUID } from "node:crypto";
2
2
  import { classifyProviderError } from "./errors.js";
3
3
  import type {
4
- GenerationOutcome,
5
- GenerationRecord,
6
- ModelIdentity,
7
- ProviderErrorRecord,
8
- RunOutcome,
9
- SettledRun,
10
- SkillActivationRecord,
11
- ToolCallRecord,
12
- TriggerSource,
4
+ GenerationOutcome,
5
+ GenerationRecord,
6
+ ModelIdentity,
7
+ ProviderErrorRecord,
8
+ RunOutcome,
9
+ SettledRun,
10
+ SkillActivationRecord,
11
+ ToolCallRecord,
12
+ TriggerSource,
13
13
  } from "./types.js";
14
14
 
15
15
  interface ActiveRun {
16
- id: string;
17
- startedAtMs: number;
18
- triggerSource: TriggerSource;
19
- initialModel?: ModelIdentity;
20
- attemptCount: number;
21
- generations: GenerationRecord[];
22
- generationIds: Set<string>;
23
- tools: ToolCallRecord[];
24
- toolIds: Set<string>;
25
- skills: SkillActivationRecord[];
26
- skillIndexes: Map<string, number>;
27
- providerErrors: ProviderErrorRecord[];
16
+ id: string;
17
+ startedAtMs: number;
18
+ triggerSource: TriggerSource;
19
+ initialModel?: ModelIdentity;
20
+ attemptCount: number;
21
+ generations: GenerationRecord[];
22
+ generationIds: Set<string>;
23
+ tools: ToolCallRecord[];
24
+ toolIds: Set<string>;
25
+ skills: SkillActivationRecord[];
26
+ skillIndexes: Map<string, number>;
27
+ providerErrors: ProviderErrorRecord[];
28
28
  }
29
29
 
30
30
  export class ResponseCollector {
31
- private active: ActiveRun | undefined;
31
+ private active: ActiveRun | undefined;
32
32
 
33
- hasActiveRun(): boolean {
34
- return this.active !== undefined;
35
- }
33
+ hasActiveRun(): boolean {
34
+ return this.active !== undefined;
35
+ }
36
36
 
37
- getActiveRunId(): string | undefined {
38
- return this.active?.id;
39
- }
37
+ getActiveRunId(): string | undefined {
38
+ return this.active?.id;
39
+ }
40
40
 
41
- begin(input: {
42
- id: string;
43
- now: number;
44
- triggerSource: TriggerSource;
45
- model?: ModelIdentity;
46
- }): SettledRun | undefined {
47
- const interrupted = this.active ? this.finalize(input.now, "interrupted") : undefined;
48
- this.active = {
49
- id: input.id,
50
- startedAtMs: input.now,
51
- triggerSource: input.triggerSource,
52
- initialModel: input.model,
53
- attemptCount: 0,
54
- generations: [],
55
- generationIds: new Set(),
56
- tools: [],
57
- toolIds: new Set(),
58
- skills: [],
59
- skillIndexes: new Map(),
60
- providerErrors: [],
61
- };
62
- return interrupted;
63
- }
41
+ begin(input: {
42
+ id: string;
43
+ now: number;
44
+ triggerSource: TriggerSource;
45
+ model?: ModelIdentity;
46
+ }): SettledRun | undefined {
47
+ const interrupted = this.active ? this.finalize(input.now, "interrupted") : undefined;
48
+ this.active = {
49
+ id: input.id,
50
+ startedAtMs: input.now,
51
+ triggerSource: input.triggerSource,
52
+ initialModel: input.model,
53
+ attemptCount: 0,
54
+ generations: [],
55
+ generationIds: new Set(),
56
+ tools: [],
57
+ toolIds: new Set(),
58
+ skills: [],
59
+ skillIndexes: new Map(),
60
+ providerErrors: [],
61
+ };
62
+ return interrupted;
63
+ }
64
64
 
65
- beginAttempt(): void {
66
- if (this.active) this.active.attemptCount += 1;
67
- }
65
+ beginAttempt(): void {
66
+ if (this.active) this.active.attemptCount += 1;
67
+ }
68
68
 
69
- beginGeneration(input: { id: string; now: number; model?: ModelIdentity }): void {
70
- const active = this.active;
71
- if (!active || active.generationIds.has(input.id)) return;
72
- active.generationIds.add(input.id);
73
- active.generations.push({
74
- id: input.id,
75
- ordinal: active.generations.length,
76
- provider: input.model?.provider,
77
- model: input.model?.model,
78
- thinkingLevel: input.model?.thinkingLevel,
79
- startedAtMs: input.now,
80
- outcome: "pending",
81
- responses: [],
82
- });
83
- }
69
+ beginGeneration(input: { id: string; now: number; model?: ModelIdentity }): void {
70
+ const active = this.active;
71
+ if (!active || active.generationIds.has(input.id)) return;
72
+ active.generationIds.add(input.id);
73
+ active.generations.push({
74
+ id: input.id,
75
+ ordinal: active.generations.length,
76
+ provider: input.model?.provider,
77
+ model: input.model?.model,
78
+ thinkingLevel: input.model?.thinkingLevel,
79
+ startedAtMs: input.now,
80
+ outcome: "pending",
81
+ responses: [],
82
+ });
83
+ }
84
84
 
85
- recordProviderResponse(input: { status: number; now: number }): void {
86
- const generation = this.latestGeneration();
87
- if (generation?.outcome !== "pending") return;
88
- generation.responses.push({
89
- ordinal: generation.responses.length,
90
- occurredAtMs: input.now,
91
- status: input.status,
92
- });
93
- }
85
+ recordProviderResponse(input: { status: number; now: number }): void {
86
+ const generation = this.latestGeneration();
87
+ if (generation?.outcome !== "pending") return;
88
+ generation.responses.push({
89
+ ordinal: generation.responses.length,
90
+ occurredAtMs: input.now,
91
+ status: input.status,
92
+ });
93
+ }
94
94
 
95
- finishGeneration(input: { now: number; stopReason: string; errorMessage?: string }): void {
96
- const active = this.active;
97
- const generation = this.latestGeneration();
98
- if (!active || !generation || generation.outcome !== "pending") return;
99
- generation.finishedAtMs = input.now;
100
- generation.durationMs = elapsed(generation.startedAtMs, input.now);
101
- generation.stopReason = input.stopReason;
102
- generation.outcome = generationOutcome(input.stopReason);
103
- if (generation.outcome === "error") {
104
- active.providerErrors.push({
105
- id: randomUUID(),
106
- generationId: generation.id,
107
- occurredAtMs: input.now,
108
- provider: generation.provider,
109
- model: generation.model,
110
- category: classifyProviderError(input.errorMessage),
111
- recovered: false,
112
- terminal: true,
113
- });
114
- }
115
- }
95
+ finishGeneration(input: { now: number; stopReason: string; errorMessage?: string }): void {
96
+ const active = this.active;
97
+ const generation = this.latestGeneration();
98
+ if (!active || !generation || generation.outcome !== "pending") return;
99
+ generation.finishedAtMs = input.now;
100
+ generation.durationMs = elapsed(generation.startedAtMs, input.now);
101
+ generation.stopReason = input.stopReason;
102
+ generation.outcome = generationOutcome(input.stopReason);
103
+ if (generation.outcome === "error") {
104
+ active.providerErrors.push({
105
+ id: randomUUID(),
106
+ generationId: generation.id,
107
+ occurredAtMs: input.now,
108
+ provider: generation.provider,
109
+ model: generation.model,
110
+ category: classifyProviderError(input.errorMessage),
111
+ recovered: false,
112
+ terminal: true,
113
+ });
114
+ }
115
+ }
116
116
 
117
- beginTool(input: { id: string; name: string; now: number; model?: ModelIdentity }): void {
118
- const active = this.active;
119
- if (!active || active.toolIds.has(input.id)) return;
120
- active.toolIds.add(input.id);
121
- active.tools.push({
122
- id: input.id,
123
- ordinal: active.tools.length,
124
- name: input.name,
125
- provider: input.model?.provider,
126
- model: input.model?.model,
127
- startedAtMs: input.now,
128
- isError: false,
129
- completionState: "running",
130
- });
131
- }
117
+ beginTool(input: { id: string; name: string; now: number; model?: ModelIdentity }): void {
118
+ const active = this.active;
119
+ if (!active || active.toolIds.has(input.id)) return;
120
+ active.toolIds.add(input.id);
121
+ active.tools.push({
122
+ id: input.id,
123
+ ordinal: active.tools.length,
124
+ name: input.name,
125
+ provider: input.model?.provider,
126
+ model: input.model?.model,
127
+ startedAtMs: input.now,
128
+ isError: false,
129
+ completionState: "running",
130
+ });
131
+ }
132
132
 
133
- finishTool(input: { id: string; now: number; isError: boolean }): void {
134
- const tool = this.active?.tools.find(({ id }) => id === input.id);
135
- if (tool?.completionState !== "running") return;
136
- tool.finishedAtMs = input.now;
137
- tool.durationMs = elapsed(tool.startedAtMs, input.now);
138
- tool.isError = input.isError;
139
- tool.completionState = "finished";
140
- }
133
+ finishTool(input: { id: string; now: number; isError: boolean }): void {
134
+ const tool = this.active?.tools.find(({ id }) => id === input.id);
135
+ if (tool?.completionState !== "running") return;
136
+ tool.finishedAtMs = input.now;
137
+ tool.durationMs = elapsed(tool.startedAtMs, input.now);
138
+ tool.isError = input.isError;
139
+ tool.completionState = "finished";
140
+ }
141
141
 
142
- activateSkill(input: {
143
- name: string;
144
- initiatedBy: "user" | "model";
145
- now: number;
146
- model?: ModelIdentity;
147
- }): void {
148
- const active = this.active;
149
- if (!active) return;
150
- const existingIndex = active.skillIndexes.get(input.name);
151
- if (existingIndex !== undefined) {
152
- const existing = active.skills[existingIndex];
153
- if (existing && existing.initiatedBy === "model" && input.initiatedBy === "user") {
154
- existing.initiatedBy = "user";
155
- existing.occurredAtMs = input.now;
156
- existing.provider = input.model?.provider;
157
- existing.model = input.model?.model;
158
- }
159
- return;
160
- }
161
- active.skillIndexes.set(input.name, active.skills.length);
162
- active.skills.push({
163
- id: randomUUID(),
164
- name: input.name,
165
- initiatedBy: input.initiatedBy,
166
- occurredAtMs: input.now,
167
- provider: input.model?.provider,
168
- model: input.model?.model,
169
- });
170
- }
142
+ activateSkill(input: { name: string; initiatedBy: "user" | "model"; now: number; model?: ModelIdentity }): void {
143
+ const active = this.active;
144
+ if (!active) return;
145
+ const existingIndex = active.skillIndexes.get(input.name);
146
+ if (existingIndex !== undefined) {
147
+ const existing = active.skills[existingIndex];
148
+ if (existing && existing.initiatedBy === "model" && input.initiatedBy === "user") {
149
+ existing.initiatedBy = "user";
150
+ existing.occurredAtMs = input.now;
151
+ existing.provider = input.model?.provider;
152
+ existing.model = input.model?.model;
153
+ }
154
+ return;
155
+ }
156
+ active.skillIndexes.set(input.name, active.skills.length);
157
+ active.skills.push({
158
+ id: randomUUID(),
159
+ name: input.name,
160
+ initiatedBy: input.initiatedBy,
161
+ occurredAtMs: input.now,
162
+ provider: input.model?.provider,
163
+ model: input.model?.model,
164
+ });
165
+ }
171
166
 
172
- settle(now: number): SettledRun | undefined {
173
- return this.finalize(now);
174
- }
167
+ settle(now: number): SettledRun | undefined {
168
+ return this.finalize(now);
169
+ }
175
170
 
176
- interrupt(now: number): SettledRun | undefined {
177
- return this.finalize(now, "interrupted");
178
- }
171
+ interrupt(now: number): SettledRun | undefined {
172
+ return this.finalize(now, "interrupted");
173
+ }
179
174
 
180
- private latestGeneration(): GenerationRecord | undefined {
181
- return this.active?.generations.at(-1);
182
- }
175
+ private latestGeneration(): GenerationRecord | undefined {
176
+ return this.active?.generations.at(-1);
177
+ }
183
178
 
184
- private finalize(now: number, forcedOutcome?: RunOutcome): SettledRun | undefined {
185
- const active = this.active;
186
- if (!active) return undefined;
187
- this.active = undefined;
179
+ private finalize(now: number, forcedOutcome?: RunOutcome): SettledRun | undefined {
180
+ const active = this.active;
181
+ if (!active) return undefined;
182
+ this.active = undefined;
188
183
 
189
- for (const generation of active.generations) {
190
- if (generation.outcome !== "pending") continue;
191
- generation.outcome = "interrupted";
192
- generation.finishedAtMs = now;
193
- generation.durationMs = elapsed(generation.startedAtMs, now);
194
- }
195
- for (const tool of active.tools) {
196
- if (tool.completionState !== "running") continue;
197
- tool.completionState = "interrupted";
198
- tool.finishedAtMs = now;
199
- tool.durationMs = elapsed(tool.startedAtMs, now);
200
- }
184
+ for (const generation of active.generations) {
185
+ if (generation.outcome !== "pending") continue;
186
+ generation.outcome = "interrupted";
187
+ generation.finishedAtMs = now;
188
+ generation.durationMs = elapsed(generation.startedAtMs, now);
189
+ }
190
+ for (const tool of active.tools) {
191
+ if (tool.completionState !== "running") continue;
192
+ tool.completionState = "interrupted";
193
+ tool.finishedAtMs = now;
194
+ tool.durationMs = elapsed(tool.startedAtMs, now);
195
+ }
201
196
 
202
- const successfulGenerationIndexes = new Set(
203
- active.generations
204
- .map((generation, index) => ({ generation, index }))
205
- .filter(({ generation }) => isSuccessfulGeneration(generation))
206
- .map(({ index }) => index),
207
- );
208
- let recoveredHttpErrors = 0;
209
- let httpErrors = 0;
210
- for (const [generationIndex, generation] of active.generations.entries()) {
211
- const hasLaterSuccess = [...successfulGenerationIndexes].some(
212
- (index) => index > generationIndex,
213
- );
214
- for (const [responseIndex, response] of generation.responses.entries()) {
215
- if (response.status < 400) continue;
216
- httpErrors += 1;
217
- const laterSuccessInGeneration = generation.responses
218
- .slice(responseIndex + 1)
219
- .some(({ status }) => status >= 200 && status < 400);
220
- if (laterSuccessInGeneration || hasLaterSuccess) recoveredHttpErrors += 1;
221
- }
222
- }
223
- for (const error of active.providerErrors) {
224
- const generationIndex = active.generations.findIndex(({ id }) => id === error.generationId);
225
- error.recovered = [...successfulGenerationIndexes].some((index) => index > generationIndex);
226
- error.terminal = !error.recovered;
227
- }
197
+ const successfulGenerationIndexes = new Set(
198
+ active.generations
199
+ .map((generation, index) => ({ generation, index }))
200
+ .filter(({ generation }) => isSuccessfulGeneration(generation))
201
+ .map(({ index }) => index),
202
+ );
203
+ let recoveredHttpErrors = 0;
204
+ let httpErrors = 0;
205
+ for (const [generationIndex, generation] of active.generations.entries()) {
206
+ const hasLaterSuccess = [...successfulGenerationIndexes].some((index) => index > generationIndex);
207
+ for (const [responseIndex, response] of generation.responses.entries()) {
208
+ if (response.status < 400) continue;
209
+ httpErrors += 1;
210
+ const laterSuccessInGeneration = generation.responses
211
+ .slice(responseIndex + 1)
212
+ .some(({ status }) => status >= 200 && status < 400);
213
+ if (laterSuccessInGeneration || hasLaterSuccess) recoveredHttpErrors += 1;
214
+ }
215
+ }
216
+ for (const error of active.providerErrors) {
217
+ const generationIndex = active.generations.findIndex(({ id }) => id === error.generationId);
218
+ error.recovered = [...successfulGenerationIndexes].some((index) => index > generationIndex);
219
+ error.terminal = !error.recovered;
220
+ }
228
221
 
229
- const recoveredGenerationErrors = active.providerErrors.filter(
230
- ({ recovered }) => recovered,
231
- ).length;
232
- const providerErrorCount = httpErrors + active.providerErrors.length;
233
- const recoveredErrorCount = recoveredHttpErrors + recoveredGenerationErrors;
234
- const outcome = forcedOutcome ?? deriveOutcome(active.generations, providerErrorCount);
222
+ const recoveredGenerationErrors = active.providerErrors.filter(({ recovered }) => recovered).length;
223
+ const providerErrorCount = httpErrors + active.providerErrors.length;
224
+ const recoveredErrorCount = recoveredHttpErrors + recoveredGenerationErrors;
225
+ const outcome = forcedOutcome ?? deriveOutcome(active.generations, providerErrorCount);
235
226
 
236
- return {
237
- id: active.id,
238
- startedAtMs: active.startedAtMs,
239
- finishedAtMs: now,
240
- durationMs: elapsed(active.startedAtMs, now),
241
- triggerSource: active.triggerSource,
242
- initialProvider: active.initialModel?.provider,
243
- initialModel: active.initialModel?.model,
244
- outcome,
245
- attemptCount: active.attemptCount,
246
- generations: active.generations,
247
- tools: active.tools,
248
- skills: active.skills,
249
- providerErrors: active.providerErrors,
250
- toolErrorCount: active.tools.filter(({ isError }) => isError).length,
251
- providerErrorCount,
252
- recoveredErrorCount,
253
- };
254
- }
227
+ return {
228
+ id: active.id,
229
+ startedAtMs: active.startedAtMs,
230
+ finishedAtMs: now,
231
+ durationMs: elapsed(active.startedAtMs, now),
232
+ triggerSource: active.triggerSource,
233
+ initialProvider: active.initialModel?.provider,
234
+ initialModel: active.initialModel?.model,
235
+ outcome,
236
+ attemptCount: active.attemptCount,
237
+ generations: active.generations,
238
+ tools: active.tools,
239
+ skills: active.skills,
240
+ providerErrors: active.providerErrors,
241
+ toolErrorCount: active.tools.filter(({ isError }) => isError).length,
242
+ providerErrorCount,
243
+ recoveredErrorCount,
244
+ };
245
+ }
255
246
  }
256
247
 
257
248
  function elapsed(start: number, end: number): number {
258
- return Math.max(0, end - start);
249
+ return Math.max(0, end - start);
259
250
  }
260
251
 
261
252
  function generationOutcome(stopReason: string): GenerationOutcome {
262
- switch (stopReason) {
263
- case "stop":
264
- return "stop";
265
- case "toolUse":
266
- return "tool_use";
267
- case "error":
268
- return "error";
269
- case "aborted":
270
- return "aborted";
271
- case "length":
272
- return "length";
273
- default:
274
- return "interrupted";
275
- }
253
+ switch (stopReason) {
254
+ case "stop":
255
+ return "stop";
256
+ case "toolUse":
257
+ return "tool_use";
258
+ case "error":
259
+ return "error";
260
+ case "aborted":
261
+ return "aborted";
262
+ case "length":
263
+ return "length";
264
+ default:
265
+ return "interrupted";
266
+ }
276
267
  }
277
268
 
278
269
  function isSuccessfulGeneration(generation: GenerationRecord): boolean {
279
- return generation.outcome === "stop" || generation.outcome === "tool_use";
270
+ return generation.outcome === "stop" || generation.outcome === "tool_use";
280
271
  }
281
272
 
282
- function deriveOutcome(
283
- generations: readonly GenerationRecord[],
284
- providerErrors: number,
285
- ): RunOutcome {
286
- const last = generations.at(-1);
287
- if (!last) return providerErrors > 0 ? "error" : "success";
288
- switch (last.outcome) {
289
- case "stop":
290
- case "tool_use":
291
- return providerErrors > 0 ? "recovered_success" : "success";
292
- case "error":
293
- return "error";
294
- case "aborted":
295
- return "aborted";
296
- case "length":
297
- return "length";
298
- default:
299
- return "interrupted";
300
- }
273
+ function deriveOutcome(generations: readonly GenerationRecord[], providerErrors: number): RunOutcome {
274
+ const last = generations.at(-1);
275
+ if (!last) return providerErrors > 0 ? "error" : "success";
276
+ switch (last.outcome) {
277
+ case "stop":
278
+ case "tool_use":
279
+ return providerErrors > 0 ? "recovered_success" : "success";
280
+ case "error":
281
+ return "error";
282
+ case "aborted":
283
+ return "aborted";
284
+ case "length":
285
+ return "length";
286
+ default:
287
+ return "interrupted";
288
+ }
301
289
  }
package/src/errors.ts CHANGED
@@ -1,16 +1,16 @@
1
1
  import type { ProviderErrorCategory } from "./types.js";
2
2
 
3
3
  export function classifyProviderError(message: string | undefined): ProviderErrorCategory {
4
- const value = message?.toLowerCase() ?? "";
5
- if (/\b(enotfound|eai_again|dns|getaddrinfo)\b/u.test(value)) return "dns";
6
- if (/\b(etimedout|timeout|timed out)\b/u.test(value)) return "timeout";
7
- if (/\b(econnrefused|connection refused)\b/u.test(value)) return "connection_refused";
8
- if (/\b(econnreset|connection reset|socket hang up)\b/u.test(value)) {
9
- return "connection_reset";
10
- }
11
- if (/\b(tls|ssl|certificate|cert_|handshake)\b/u.test(value)) return "tls";
12
- if (/\b(fetch failed|network|socket|connection|transport)\b/u.test(value)) {
13
- return "network_other";
14
- }
15
- return "provider_other";
4
+ const value = message?.toLowerCase() ?? "";
5
+ if (/\b(enotfound|eai_again|dns|getaddrinfo)\b/u.test(value)) return "dns";
6
+ if (/\b(etimedout|timeout|timed out)\b/u.test(value)) return "timeout";
7
+ if (/\b(econnrefused|connection refused)\b/u.test(value)) return "connection_refused";
8
+ if (/\b(econnreset|connection reset|socket hang up)\b/u.test(value)) {
9
+ return "connection_reset";
10
+ }
11
+ if (/\b(tls|ssl|certificate|cert_|handshake)\b/u.test(value)) return "tls";
12
+ if (/\b(fetch failed|network|socket|connection|transport)\b/u.test(value)) {
13
+ return "network_other";
14
+ }
15
+ return "provider_other";
16
16
  }