@narumitw/pi-analytics 0.49.8 → 0.49.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -36
- package/dist/index.ts +53 -106
- package/dist/index.ts.map +2 -2
- package/package.json +49 -50
- package/src/analytics.ts +368 -358
- package/src/collector.ts +250 -262
- package/src/errors.ts +12 -12
- package/src/menu.ts +325 -343
- package/src/skills.ts +55 -57
- package/src/storage/files.ts +352 -374
- package/src/storage/format.ts +159 -201
- package/src/storage/queries.ts +200 -204
- package/src/storage/store.ts +39 -43
- package/src/types.ts +66 -79
package/src/collector.ts
CHANGED
|
@@ -1,301 +1,289 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { classifyProviderError } from "./errors.js";
|
|
3
3
|
import type {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
4
|
+
GenerationOutcome,
|
|
5
|
+
GenerationRecord,
|
|
6
|
+
ModelIdentity,
|
|
7
|
+
ProviderErrorRecord,
|
|
8
|
+
RunOutcome,
|
|
9
|
+
SettledRun,
|
|
10
|
+
SkillActivationRecord,
|
|
11
|
+
ToolCallRecord,
|
|
12
|
+
TriggerSource,
|
|
13
13
|
} from "./types.js";
|
|
14
14
|
|
|
15
15
|
interface ActiveRun {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
16
|
+
id: string;
|
|
17
|
+
startedAtMs: number;
|
|
18
|
+
triggerSource: TriggerSource;
|
|
19
|
+
initialModel?: ModelIdentity;
|
|
20
|
+
attemptCount: number;
|
|
21
|
+
generations: GenerationRecord[];
|
|
22
|
+
generationIds: Set<string>;
|
|
23
|
+
tools: ToolCallRecord[];
|
|
24
|
+
toolIds: Set<string>;
|
|
25
|
+
skills: SkillActivationRecord[];
|
|
26
|
+
skillIndexes: Map<string, number>;
|
|
27
|
+
providerErrors: ProviderErrorRecord[];
|
|
28
28
|
}
|
|
29
29
|
|
|
30
30
|
export class ResponseCollector {
|
|
31
|
-
|
|
31
|
+
private active: ActiveRun | undefined;
|
|
32
32
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
33
|
+
hasActiveRun(): boolean {
|
|
34
|
+
return this.active !== undefined;
|
|
35
|
+
}
|
|
36
36
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
37
|
+
getActiveRunId(): string | undefined {
|
|
38
|
+
return this.active?.id;
|
|
39
|
+
}
|
|
40
40
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
41
|
+
begin(input: {
|
|
42
|
+
id: string;
|
|
43
|
+
now: number;
|
|
44
|
+
triggerSource: TriggerSource;
|
|
45
|
+
model?: ModelIdentity;
|
|
46
|
+
}): SettledRun | undefined {
|
|
47
|
+
const interrupted = this.active ? this.finalize(input.now, "interrupted") : undefined;
|
|
48
|
+
this.active = {
|
|
49
|
+
id: input.id,
|
|
50
|
+
startedAtMs: input.now,
|
|
51
|
+
triggerSource: input.triggerSource,
|
|
52
|
+
initialModel: input.model,
|
|
53
|
+
attemptCount: 0,
|
|
54
|
+
generations: [],
|
|
55
|
+
generationIds: new Set(),
|
|
56
|
+
tools: [],
|
|
57
|
+
toolIds: new Set(),
|
|
58
|
+
skills: [],
|
|
59
|
+
skillIndexes: new Map(),
|
|
60
|
+
providerErrors: [],
|
|
61
|
+
};
|
|
62
|
+
return interrupted;
|
|
63
|
+
}
|
|
64
64
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
65
|
+
beginAttempt(): void {
|
|
66
|
+
if (this.active) this.active.attemptCount += 1;
|
|
67
|
+
}
|
|
68
68
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
69
|
+
beginGeneration(input: { id: string; now: number; model?: ModelIdentity }): void {
|
|
70
|
+
const active = this.active;
|
|
71
|
+
if (!active || active.generationIds.has(input.id)) return;
|
|
72
|
+
active.generationIds.add(input.id);
|
|
73
|
+
active.generations.push({
|
|
74
|
+
id: input.id,
|
|
75
|
+
ordinal: active.generations.length,
|
|
76
|
+
provider: input.model?.provider,
|
|
77
|
+
model: input.model?.model,
|
|
78
|
+
thinkingLevel: input.model?.thinkingLevel,
|
|
79
|
+
startedAtMs: input.now,
|
|
80
|
+
outcome: "pending",
|
|
81
|
+
responses: [],
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
84
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
85
|
+
recordProviderResponse(input: { status: number; now: number }): void {
|
|
86
|
+
const generation = this.latestGeneration();
|
|
87
|
+
if (generation?.outcome !== "pending") return;
|
|
88
|
+
generation.responses.push({
|
|
89
|
+
ordinal: generation.responses.length,
|
|
90
|
+
occurredAtMs: input.now,
|
|
91
|
+
status: input.status,
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
94
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
95
|
+
finishGeneration(input: { now: number; stopReason: string; errorMessage?: string }): void {
|
|
96
|
+
const active = this.active;
|
|
97
|
+
const generation = this.latestGeneration();
|
|
98
|
+
if (!active || !generation || generation.outcome !== "pending") return;
|
|
99
|
+
generation.finishedAtMs = input.now;
|
|
100
|
+
generation.durationMs = elapsed(generation.startedAtMs, input.now);
|
|
101
|
+
generation.stopReason = input.stopReason;
|
|
102
|
+
generation.outcome = generationOutcome(input.stopReason);
|
|
103
|
+
if (generation.outcome === "error") {
|
|
104
|
+
active.providerErrors.push({
|
|
105
|
+
id: randomUUID(),
|
|
106
|
+
generationId: generation.id,
|
|
107
|
+
occurredAtMs: input.now,
|
|
108
|
+
provider: generation.provider,
|
|
109
|
+
model: generation.model,
|
|
110
|
+
category: classifyProviderError(input.errorMessage),
|
|
111
|
+
recovered: false,
|
|
112
|
+
terminal: true,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
116
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
117
|
+
beginTool(input: { id: string; name: string; now: number; model?: ModelIdentity }): void {
|
|
118
|
+
const active = this.active;
|
|
119
|
+
if (!active || active.toolIds.has(input.id)) return;
|
|
120
|
+
active.toolIds.add(input.id);
|
|
121
|
+
active.tools.push({
|
|
122
|
+
id: input.id,
|
|
123
|
+
ordinal: active.tools.length,
|
|
124
|
+
name: input.name,
|
|
125
|
+
provider: input.model?.provider,
|
|
126
|
+
model: input.model?.model,
|
|
127
|
+
startedAtMs: input.now,
|
|
128
|
+
isError: false,
|
|
129
|
+
completionState: "running",
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
132
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
133
|
+
finishTool(input: { id: string; now: number; isError: boolean }): void {
|
|
134
|
+
const tool = this.active?.tools.find(({ id }) => id === input.id);
|
|
135
|
+
if (tool?.completionState !== "running") return;
|
|
136
|
+
tool.finishedAtMs = input.now;
|
|
137
|
+
tool.durationMs = elapsed(tool.startedAtMs, input.now);
|
|
138
|
+
tool.isError = input.isError;
|
|
139
|
+
tool.completionState = "finished";
|
|
140
|
+
}
|
|
141
141
|
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
occurredAtMs: input.now,
|
|
167
|
-
provider: input.model?.provider,
|
|
168
|
-
model: input.model?.model,
|
|
169
|
-
});
|
|
170
|
-
}
|
|
142
|
+
activateSkill(input: { name: string; initiatedBy: "user" | "model"; now: number; model?: ModelIdentity }): void {
|
|
143
|
+
const active = this.active;
|
|
144
|
+
if (!active) return;
|
|
145
|
+
const existingIndex = active.skillIndexes.get(input.name);
|
|
146
|
+
if (existingIndex !== undefined) {
|
|
147
|
+
const existing = active.skills[existingIndex];
|
|
148
|
+
if (existing && existing.initiatedBy === "model" && input.initiatedBy === "user") {
|
|
149
|
+
existing.initiatedBy = "user";
|
|
150
|
+
existing.occurredAtMs = input.now;
|
|
151
|
+
existing.provider = input.model?.provider;
|
|
152
|
+
existing.model = input.model?.model;
|
|
153
|
+
}
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
active.skillIndexes.set(input.name, active.skills.length);
|
|
157
|
+
active.skills.push({
|
|
158
|
+
id: randomUUID(),
|
|
159
|
+
name: input.name,
|
|
160
|
+
initiatedBy: input.initiatedBy,
|
|
161
|
+
occurredAtMs: input.now,
|
|
162
|
+
provider: input.model?.provider,
|
|
163
|
+
model: input.model?.model,
|
|
164
|
+
});
|
|
165
|
+
}
|
|
171
166
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
167
|
+
settle(now: number): SettledRun | undefined {
|
|
168
|
+
return this.finalize(now);
|
|
169
|
+
}
|
|
175
170
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
171
|
+
interrupt(now: number): SettledRun | undefined {
|
|
172
|
+
return this.finalize(now, "interrupted");
|
|
173
|
+
}
|
|
179
174
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
175
|
+
private latestGeneration(): GenerationRecord | undefined {
|
|
176
|
+
return this.active?.generations.at(-1);
|
|
177
|
+
}
|
|
183
178
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
179
|
+
private finalize(now: number, forcedOutcome?: RunOutcome): SettledRun | undefined {
|
|
180
|
+
const active = this.active;
|
|
181
|
+
if (!active) return undefined;
|
|
182
|
+
this.active = undefined;
|
|
188
183
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
184
|
+
for (const generation of active.generations) {
|
|
185
|
+
if (generation.outcome !== "pending") continue;
|
|
186
|
+
generation.outcome = "interrupted";
|
|
187
|
+
generation.finishedAtMs = now;
|
|
188
|
+
generation.durationMs = elapsed(generation.startedAtMs, now);
|
|
189
|
+
}
|
|
190
|
+
for (const tool of active.tools) {
|
|
191
|
+
if (tool.completionState !== "running") continue;
|
|
192
|
+
tool.completionState = "interrupted";
|
|
193
|
+
tool.finishedAtMs = now;
|
|
194
|
+
tool.durationMs = elapsed(tool.startedAtMs, now);
|
|
195
|
+
}
|
|
201
196
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
error.terminal = !error.recovered;
|
|
227
|
-
}
|
|
197
|
+
const successfulGenerationIndexes = new Set(
|
|
198
|
+
active.generations
|
|
199
|
+
.map((generation, index) => ({ generation, index }))
|
|
200
|
+
.filter(({ generation }) => isSuccessfulGeneration(generation))
|
|
201
|
+
.map(({ index }) => index),
|
|
202
|
+
);
|
|
203
|
+
let recoveredHttpErrors = 0;
|
|
204
|
+
let httpErrors = 0;
|
|
205
|
+
for (const [generationIndex, generation] of active.generations.entries()) {
|
|
206
|
+
const hasLaterSuccess = [...successfulGenerationIndexes].some((index) => index > generationIndex);
|
|
207
|
+
for (const [responseIndex, response] of generation.responses.entries()) {
|
|
208
|
+
if (response.status < 400) continue;
|
|
209
|
+
httpErrors += 1;
|
|
210
|
+
const laterSuccessInGeneration = generation.responses
|
|
211
|
+
.slice(responseIndex + 1)
|
|
212
|
+
.some(({ status }) => status >= 200 && status < 400);
|
|
213
|
+
if (laterSuccessInGeneration || hasLaterSuccess) recoveredHttpErrors += 1;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
for (const error of active.providerErrors) {
|
|
217
|
+
const generationIndex = active.generations.findIndex(({ id }) => id === error.generationId);
|
|
218
|
+
error.recovered = [...successfulGenerationIndexes].some((index) => index > generationIndex);
|
|
219
|
+
error.terminal = !error.recovered;
|
|
220
|
+
}
|
|
228
221
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
const recoveredErrorCount = recoveredHttpErrors + recoveredGenerationErrors;
|
|
234
|
-
const outcome = forcedOutcome ?? deriveOutcome(active.generations, providerErrorCount);
|
|
222
|
+
const recoveredGenerationErrors = active.providerErrors.filter(({ recovered }) => recovered).length;
|
|
223
|
+
const providerErrorCount = httpErrors + active.providerErrors.length;
|
|
224
|
+
const recoveredErrorCount = recoveredHttpErrors + recoveredGenerationErrors;
|
|
225
|
+
const outcome = forcedOutcome ?? deriveOutcome(active.generations, providerErrorCount);
|
|
235
226
|
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
227
|
+
return {
|
|
228
|
+
id: active.id,
|
|
229
|
+
startedAtMs: active.startedAtMs,
|
|
230
|
+
finishedAtMs: now,
|
|
231
|
+
durationMs: elapsed(active.startedAtMs, now),
|
|
232
|
+
triggerSource: active.triggerSource,
|
|
233
|
+
initialProvider: active.initialModel?.provider,
|
|
234
|
+
initialModel: active.initialModel?.model,
|
|
235
|
+
outcome,
|
|
236
|
+
attemptCount: active.attemptCount,
|
|
237
|
+
generations: active.generations,
|
|
238
|
+
tools: active.tools,
|
|
239
|
+
skills: active.skills,
|
|
240
|
+
providerErrors: active.providerErrors,
|
|
241
|
+
toolErrorCount: active.tools.filter(({ isError }) => isError).length,
|
|
242
|
+
providerErrorCount,
|
|
243
|
+
recoveredErrorCount,
|
|
244
|
+
};
|
|
245
|
+
}
|
|
255
246
|
}
|
|
256
247
|
|
|
257
248
|
function elapsed(start: number, end: number): number {
|
|
258
|
-
|
|
249
|
+
return Math.max(0, end - start);
|
|
259
250
|
}
|
|
260
251
|
|
|
261
252
|
function generationOutcome(stopReason: string): GenerationOutcome {
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
253
|
+
switch (stopReason) {
|
|
254
|
+
case "stop":
|
|
255
|
+
return "stop";
|
|
256
|
+
case "toolUse":
|
|
257
|
+
return "tool_use";
|
|
258
|
+
case "error":
|
|
259
|
+
return "error";
|
|
260
|
+
case "aborted":
|
|
261
|
+
return "aborted";
|
|
262
|
+
case "length":
|
|
263
|
+
return "length";
|
|
264
|
+
default:
|
|
265
|
+
return "interrupted";
|
|
266
|
+
}
|
|
276
267
|
}
|
|
277
268
|
|
|
278
269
|
function isSuccessfulGeneration(generation: GenerationRecord): boolean {
|
|
279
|
-
|
|
270
|
+
return generation.outcome === "stop" || generation.outcome === "tool_use";
|
|
280
271
|
}
|
|
281
272
|
|
|
282
|
-
function deriveOutcome(
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
)
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
default:
|
|
299
|
-
return "interrupted";
|
|
300
|
-
}
|
|
273
|
+
function deriveOutcome(generations: readonly GenerationRecord[], providerErrors: number): RunOutcome {
|
|
274
|
+
const last = generations.at(-1);
|
|
275
|
+
if (!last) return providerErrors > 0 ? "error" : "success";
|
|
276
|
+
switch (last.outcome) {
|
|
277
|
+
case "stop":
|
|
278
|
+
case "tool_use":
|
|
279
|
+
return providerErrors > 0 ? "recovered_success" : "success";
|
|
280
|
+
case "error":
|
|
281
|
+
return "error";
|
|
282
|
+
case "aborted":
|
|
283
|
+
return "aborted";
|
|
284
|
+
case "length":
|
|
285
|
+
return "length";
|
|
286
|
+
default:
|
|
287
|
+
return "interrupted";
|
|
288
|
+
}
|
|
301
289
|
}
|
package/src/errors.ts
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
import type { ProviderErrorCategory } from "./types.js";
|
|
2
2
|
|
|
3
3
|
export function classifyProviderError(message: string | undefined): ProviderErrorCategory {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
4
|
+
const value = message?.toLowerCase() ?? "";
|
|
5
|
+
if (/\b(enotfound|eai_again|dns|getaddrinfo)\b/u.test(value)) return "dns";
|
|
6
|
+
if (/\b(etimedout|timeout|timed out)\b/u.test(value)) return "timeout";
|
|
7
|
+
if (/\b(econnrefused|connection refused)\b/u.test(value)) return "connection_refused";
|
|
8
|
+
if (/\b(econnreset|connection reset|socket hang up)\b/u.test(value)) {
|
|
9
|
+
return "connection_reset";
|
|
10
|
+
}
|
|
11
|
+
if (/\b(tls|ssl|certificate|cert_|handshake)\b/u.test(value)) return "tls";
|
|
12
|
+
if (/\b(fetch failed|network|socket|connection|transport)\b/u.test(value)) {
|
|
13
|
+
return "network_other";
|
|
14
|
+
}
|
|
15
|
+
return "provider_other";
|
|
16
16
|
}
|