@narumitw/pi-analytics 0.49.8 → 0.49.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -36
- package/dist/index.ts +53 -106
- package/dist/index.ts.map +2 -2
- package/package.json +49 -50
- package/src/analytics.ts +368 -358
- package/src/collector.ts +250 -262
- package/src/errors.ts +12 -12
- package/src/menu.ts +325 -343
- package/src/skills.ts +55 -57
- package/src/storage/files.ts +352 -374
- package/src/storage/format.ts +159 -201
- package/src/storage/queries.ts +200 -204
- package/src/storage/store.ts +39 -43
- package/src/types.ts +66 -79
package/src/storage/format.ts
CHANGED
|
@@ -1,281 +1,239 @@
|
|
|
1
1
|
import type {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
2
|
+
GenerationOutcome,
|
|
3
|
+
GenerationRecord,
|
|
4
|
+
ProviderErrorCategory,
|
|
5
|
+
ProviderErrorRecord,
|
|
6
|
+
ProviderResponseRecord,
|
|
7
|
+
RunOutcome,
|
|
8
|
+
SettledRun,
|
|
9
|
+
SkillActivationRecord,
|
|
10
|
+
ToolCallRecord,
|
|
11
11
|
} from "../types.js";
|
|
12
12
|
|
|
13
13
|
export const MAX_STORED_RUN_BYTES = 1024 * 1024;
|
|
14
14
|
const MAX_STRING_LENGTH = 4096;
|
|
15
15
|
const MAX_NESTED_RECORDS = 20_000;
|
|
16
16
|
const TRIGGER_SOURCES = ["interactive", "rpc", "extension", "unknown"] as const;
|
|
17
|
-
const RUN_OUTCOMES = [
|
|
18
|
-
|
|
19
|
-
"recovered_success",
|
|
20
|
-
"error",
|
|
21
|
-
"aborted",
|
|
22
|
-
"length",
|
|
23
|
-
"interrupted",
|
|
24
|
-
] as const;
|
|
25
|
-
const GENERATION_OUTCOMES = [
|
|
26
|
-
"pending",
|
|
27
|
-
"stop",
|
|
28
|
-
"tool_use",
|
|
29
|
-
"error",
|
|
30
|
-
"aborted",
|
|
31
|
-
"length",
|
|
32
|
-
"interrupted",
|
|
33
|
-
] as const;
|
|
17
|
+
const RUN_OUTCOMES = ["success", "recovered_success", "error", "aborted", "length", "interrupted"] as const;
|
|
18
|
+
const GENERATION_OUTCOMES = ["pending", "stop", "tool_use", "error", "aborted", "length", "interrupted"] as const;
|
|
34
19
|
const ERROR_CATEGORIES = [
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
20
|
+
"dns",
|
|
21
|
+
"timeout",
|
|
22
|
+
"connection_refused",
|
|
23
|
+
"connection_reset",
|
|
24
|
+
"tls",
|
|
25
|
+
"network_other",
|
|
26
|
+
"provider_other",
|
|
42
27
|
] as const;
|
|
43
28
|
|
|
44
29
|
export class AnalyticsStorageFormatError extends Error {
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
30
|
+
constructor(message: string, options: { cause?: unknown } = {}) {
|
|
31
|
+
super(message, options);
|
|
32
|
+
this.name = "AnalyticsStorageFormatError";
|
|
33
|
+
}
|
|
49
34
|
}
|
|
50
35
|
|
|
51
36
|
export function encodeStoredRun(run: SettledRun): string {
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
37
|
+
const encoded = `${JSON.stringify({
|
|
38
|
+
formatVersion: 1,
|
|
39
|
+
run: parseRun(run, { remaining: MAX_NESTED_RECORDS }),
|
|
40
|
+
})}\n`;
|
|
41
|
+
if (Buffer.byteLength(encoded) > MAX_STORED_RUN_BYTES) {
|
|
42
|
+
throw new AnalyticsStorageFormatError("Analytics record is too large to store safely.");
|
|
43
|
+
}
|
|
44
|
+
return encoded;
|
|
60
45
|
}
|
|
61
46
|
|
|
62
47
|
export function decodeStoredRun(line: string): SettledRun {
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
48
|
+
if (Buffer.byteLength(line) > MAX_STORED_RUN_BYTES) {
|
|
49
|
+
throw new AnalyticsStorageFormatError("Analytics record is too large to read safely.");
|
|
50
|
+
}
|
|
51
|
+
let value: unknown;
|
|
52
|
+
try {
|
|
53
|
+
value = JSON.parse(line);
|
|
54
|
+
} catch (error) {
|
|
55
|
+
throw new AnalyticsStorageFormatError("Analytics record contains invalid JSON.", {
|
|
56
|
+
cause: error,
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
const envelope = asRecord(value, "analytics record");
|
|
60
|
+
if (envelope.formatVersion !== 1) {
|
|
61
|
+
throw new AnalyticsStorageFormatError("Analytics record uses an unsupported format version.");
|
|
62
|
+
}
|
|
63
|
+
return parseRun(envelope.run, { remaining: MAX_NESTED_RECORDS });
|
|
79
64
|
}
|
|
80
65
|
|
|
81
66
|
interface ParseBudget {
|
|
82
|
-
|
|
67
|
+
remaining: number;
|
|
83
68
|
}
|
|
84
69
|
|
|
85
70
|
function parseRun(value: unknown, budget: ParseBudget): SettledRun {
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
),
|
|
108
|
-
toolErrorCount: boundedCount(run.toolErrorCount, "run.toolErrorCount"),
|
|
109
|
-
providerErrorCount: boundedCount(run.providerErrorCount, "run.providerErrorCount"),
|
|
110
|
-
recoveredErrorCount: boundedCount(run.recoveredErrorCount, "run.recoveredErrorCount"),
|
|
111
|
-
};
|
|
71
|
+
const run = asRecord(value, "run");
|
|
72
|
+
return {
|
|
73
|
+
id: requiredString(run.id, "run.id"),
|
|
74
|
+
startedAtMs: timestampValue(run.startedAtMs, "run.startedAtMs"),
|
|
75
|
+
finishedAtMs: timestampValue(run.finishedAtMs, "run.finishedAtMs"),
|
|
76
|
+
durationMs: durationValue(run.durationMs, "run.durationMs"),
|
|
77
|
+
triggerSource: enumValue(run.triggerSource, TRIGGER_SOURCES, "run.triggerSource"),
|
|
78
|
+
...optionalProperty("initialProvider", optionalString(run.initialProvider, "run.initialProvider")),
|
|
79
|
+
...optionalProperty("initialModel", optionalString(run.initialModel, "run.initialModel")),
|
|
80
|
+
outcome: enumValue(run.outcome, RUN_OUTCOMES, "run.outcome") as RunOutcome,
|
|
81
|
+
attemptCount: boundedCount(run.attemptCount, "run.attemptCount"),
|
|
82
|
+
generations: boundedArray(run.generations, "run.generations", budget).map((item, index) =>
|
|
83
|
+
parseGeneration(item, index, budget),
|
|
84
|
+
),
|
|
85
|
+
tools: boundedArray(run.tools, "run.tools", budget).map(parseTool),
|
|
86
|
+
skills: boundedArray(run.skills, "run.skills", budget).map(parseSkill),
|
|
87
|
+
providerErrors: boundedArray(run.providerErrors, "run.providerErrors", budget).map(parseProviderError),
|
|
88
|
+
toolErrorCount: boundedCount(run.toolErrorCount, "run.toolErrorCount"),
|
|
89
|
+
providerErrorCount: boundedCount(run.providerErrorCount, "run.providerErrorCount"),
|
|
90
|
+
recoveredErrorCount: boundedCount(run.recoveredErrorCount, "run.recoveredErrorCount"),
|
|
91
|
+
};
|
|
112
92
|
}
|
|
113
93
|
|
|
114
94
|
function parseGeneration(value: unknown, index: number, budget: ParseBudget): GenerationRecord {
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
),
|
|
131
|
-
...optionalProperty("durationMs", optionalDuration(item.durationMs, `${prefix}.durationMs`)),
|
|
132
|
-
...optionalProperty("stopReason", optionalString(item.stopReason, `${prefix}.stopReason`)),
|
|
133
|
-
outcome: enumValue(item.outcome, GENERATION_OUTCOMES, `${prefix}.outcome`) as GenerationOutcome,
|
|
134
|
-
responses: boundedArray(item.responses, `${prefix}.responses`, budget).map(
|
|
135
|
-
parseProviderResponse,
|
|
136
|
-
),
|
|
137
|
-
};
|
|
95
|
+
const item = asRecord(value, `run.generations[${index}]`);
|
|
96
|
+
const prefix = `run.generations[${index}]`;
|
|
97
|
+
return {
|
|
98
|
+
id: requiredString(item.id, `${prefix}.id`),
|
|
99
|
+
ordinal: boundedCount(item.ordinal, `${prefix}.ordinal`),
|
|
100
|
+
...optionalProperty("provider", optionalString(item.provider, `${prefix}.provider`)),
|
|
101
|
+
...optionalProperty("model", optionalString(item.model, `${prefix}.model`)),
|
|
102
|
+
...optionalProperty("thinkingLevel", optionalString(item.thinkingLevel, `${prefix}.thinkingLevel`)),
|
|
103
|
+
startedAtMs: timestampValue(item.startedAtMs, `${prefix}.startedAtMs`),
|
|
104
|
+
...optionalProperty("finishedAtMs", optionalTimestamp(item.finishedAtMs, `${prefix}.finishedAtMs`)),
|
|
105
|
+
...optionalProperty("durationMs", optionalDuration(item.durationMs, `${prefix}.durationMs`)),
|
|
106
|
+
...optionalProperty("stopReason", optionalString(item.stopReason, `${prefix}.stopReason`)),
|
|
107
|
+
outcome: enumValue(item.outcome, GENERATION_OUTCOMES, `${prefix}.outcome`) as GenerationOutcome,
|
|
108
|
+
responses: boundedArray(item.responses, `${prefix}.responses`, budget).map(parseProviderResponse),
|
|
109
|
+
};
|
|
138
110
|
}
|
|
139
111
|
|
|
140
112
|
function parseProviderResponse(value: unknown, index: number): ProviderResponseRecord {
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
113
|
+
const item = asRecord(value, `provider response ${index}`);
|
|
114
|
+
return {
|
|
115
|
+
ordinal: boundedCount(item.ordinal, "providerResponse.ordinal"),
|
|
116
|
+
occurredAtMs: timestampValue(item.occurredAtMs, "providerResponse.occurredAtMs"),
|
|
117
|
+
status: boundedInteger(item.status, "providerResponse.status", 999),
|
|
118
|
+
};
|
|
147
119
|
}
|
|
148
120
|
|
|
149
121
|
function parseTool(value: unknown, index: number): ToolCallRecord {
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
`${prefix}.completionState`,
|
|
170
|
-
),
|
|
171
|
-
};
|
|
122
|
+
const item = asRecord(value, `run.tools[${index}]`);
|
|
123
|
+
const prefix = `run.tools[${index}]`;
|
|
124
|
+
const ordinal = boundedCount(item.ordinal, `${prefix}.ordinal`);
|
|
125
|
+
return {
|
|
126
|
+
id: `tool-${ordinal}`,
|
|
127
|
+
ordinal,
|
|
128
|
+
name: requiredString(item.name, `${prefix}.name`),
|
|
129
|
+
...optionalProperty("provider", optionalString(item.provider, `${prefix}.provider`)),
|
|
130
|
+
...optionalProperty("model", optionalString(item.model, `${prefix}.model`)),
|
|
131
|
+
startedAtMs: timestampValue(item.startedAtMs, `${prefix}.startedAtMs`),
|
|
132
|
+
...optionalProperty("finishedAtMs", optionalTimestamp(item.finishedAtMs, `${prefix}.finishedAtMs`)),
|
|
133
|
+
...optionalProperty("durationMs", optionalDuration(item.durationMs, `${prefix}.durationMs`)),
|
|
134
|
+
isError: booleanValue(item.isError, `${prefix}.isError`),
|
|
135
|
+
completionState: enumValue(
|
|
136
|
+
item.completionState,
|
|
137
|
+
["running", "finished", "interrupted"] as const,
|
|
138
|
+
`${prefix}.completionState`,
|
|
139
|
+
),
|
|
140
|
+
};
|
|
172
141
|
}
|
|
173
142
|
|
|
174
143
|
function parseSkill(value: unknown, index: number): SkillActivationRecord {
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
144
|
+
const item = asRecord(value, `run.skills[${index}]`);
|
|
145
|
+
const prefix = `run.skills[${index}]`;
|
|
146
|
+
return {
|
|
147
|
+
id: requiredString(item.id, `${prefix}.id`),
|
|
148
|
+
name: requiredString(item.name, `${prefix}.name`),
|
|
149
|
+
initiatedBy: enumValue(item.initiatedBy, ["user", "model"] as const, `${prefix}.initiatedBy`),
|
|
150
|
+
occurredAtMs: timestampValue(item.occurredAtMs, `${prefix}.occurredAtMs`),
|
|
151
|
+
...optionalProperty("provider", optionalString(item.provider, `${prefix}.provider`)),
|
|
152
|
+
...optionalProperty("model", optionalString(item.model, `${prefix}.model`)),
|
|
153
|
+
};
|
|
185
154
|
}
|
|
186
155
|
|
|
187
156
|
function parseProviderError(value: unknown, index: number): ProviderErrorRecord {
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
item.category,
|
|
201
|
-
ERROR_CATEGORIES,
|
|
202
|
-
`${prefix}.category`,
|
|
203
|
-
) as ProviderErrorCategory,
|
|
204
|
-
recovered: booleanValue(item.recovered, `${prefix}.recovered`),
|
|
205
|
-
terminal: booleanValue(item.terminal, `${prefix}.terminal`),
|
|
206
|
-
};
|
|
157
|
+
const item = asRecord(value, `run.providerErrors[${index}]`);
|
|
158
|
+
const prefix = `run.providerErrors[${index}]`;
|
|
159
|
+
return {
|
|
160
|
+
id: requiredString(item.id, `${prefix}.id`),
|
|
161
|
+
...optionalProperty("generationId", optionalString(item.generationId, `${prefix}.generationId`)),
|
|
162
|
+
occurredAtMs: timestampValue(item.occurredAtMs, `${prefix}.occurredAtMs`),
|
|
163
|
+
...optionalProperty("provider", optionalString(item.provider, `${prefix}.provider`)),
|
|
164
|
+
...optionalProperty("model", optionalString(item.model, `${prefix}.model`)),
|
|
165
|
+
category: enumValue(item.category, ERROR_CATEGORIES, `${prefix}.category`) as ProviderErrorCategory,
|
|
166
|
+
recovered: booleanValue(item.recovered, `${prefix}.recovered`),
|
|
167
|
+
terminal: booleanValue(item.terminal, `${prefix}.terminal`),
|
|
168
|
+
};
|
|
207
169
|
}
|
|
208
170
|
|
|
209
171
|
function asRecord(value: unknown, name: string): Record<string, unknown> {
|
|
210
|
-
|
|
211
|
-
|
|
172
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) invalid(name);
|
|
173
|
+
return value as Record<string, unknown>;
|
|
212
174
|
}
|
|
213
175
|
|
|
214
176
|
function boundedArray(value: unknown, name: string, budget: ParseBudget): unknown[] {
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
177
|
+
if (!Array.isArray(value)) invalid(name);
|
|
178
|
+
budget.remaining -= value.length;
|
|
179
|
+
if (budget.remaining < 0) {
|
|
180
|
+
throw new AnalyticsStorageFormatError("Analytics record is too large to process safely.");
|
|
181
|
+
}
|
|
182
|
+
return value;
|
|
221
183
|
}
|
|
222
184
|
|
|
223
185
|
function requiredString(value: unknown, name: string): string {
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
186
|
+
if (typeof value !== "string" || value.length === 0 || value.length > MAX_STRING_LENGTH) {
|
|
187
|
+
invalid(name);
|
|
188
|
+
}
|
|
189
|
+
return value;
|
|
228
190
|
}
|
|
229
191
|
|
|
230
192
|
function optionalString(value: unknown, name: string): string | undefined {
|
|
231
|
-
|
|
193
|
+
return value === undefined ? undefined : requiredString(value, name);
|
|
232
194
|
}
|
|
233
195
|
|
|
234
196
|
function boundedInteger(value: unknown, name: string, maximum: number): number {
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
197
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0 || value > maximum) {
|
|
198
|
+
invalid(name);
|
|
199
|
+
}
|
|
200
|
+
return value;
|
|
239
201
|
}
|
|
240
202
|
|
|
241
203
|
function timestampValue(value: unknown, name: string): number {
|
|
242
|
-
|
|
204
|
+
return boundedInteger(value, name, Number.MAX_SAFE_INTEGER);
|
|
243
205
|
}
|
|
244
206
|
|
|
245
207
|
function optionalTimestamp(value: unknown, name: string): number | undefined {
|
|
246
|
-
|
|
208
|
+
return value === undefined ? undefined : timestampValue(value, name);
|
|
247
209
|
}
|
|
248
210
|
|
|
249
211
|
function durationValue(value: unknown, name: string): number {
|
|
250
|
-
|
|
212
|
+
return boundedInteger(value, name, Math.floor(Number.MAX_SAFE_INTEGER / MAX_NESTED_RECORDS));
|
|
251
213
|
}
|
|
252
214
|
|
|
253
215
|
function optionalDuration(value: unknown, name: string): number | undefined {
|
|
254
|
-
|
|
216
|
+
return value === undefined ? undefined : durationValue(value, name);
|
|
255
217
|
}
|
|
256
218
|
|
|
257
219
|
function boundedCount(value: unknown, name: string): number {
|
|
258
|
-
|
|
220
|
+
return boundedInteger(value, name, MAX_NESTED_RECORDS);
|
|
259
221
|
}
|
|
260
222
|
|
|
261
223
|
function booleanValue(value: unknown, name: string): boolean {
|
|
262
|
-
|
|
263
|
-
|
|
224
|
+
if (typeof value !== "boolean") invalid(name);
|
|
225
|
+
return value;
|
|
264
226
|
}
|
|
265
227
|
|
|
266
|
-
function enumValue<const T extends readonly string[]>(
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
name: string,
|
|
270
|
-
): T[number] {
|
|
271
|
-
if (typeof value !== "string" || !values.includes(value)) invalid(name);
|
|
272
|
-
return value as T[number];
|
|
228
|
+
function enumValue<const T extends readonly string[]>(value: unknown, values: T, name: string): T[number] {
|
|
229
|
+
if (typeof value !== "string" || !values.includes(value)) invalid(name);
|
|
230
|
+
return value as T[number];
|
|
273
231
|
}
|
|
274
232
|
|
|
275
233
|
function optionalProperty<K extends string, T>(key: K, value: T | undefined): { [P in K]?: T } {
|
|
276
|
-
|
|
234
|
+
return value === undefined ? {} : ({ [key]: value } as { [P in K]?: T });
|
|
277
235
|
}
|
|
278
236
|
|
|
279
237
|
function invalid(name: string): never {
|
|
280
|
-
|
|
238
|
+
throw new AnalyticsStorageFormatError(`Analytics record has an invalid ${name}.`);
|
|
281
239
|
}
|