@mastra/longmemeval 0.0.0-error-handler-fix-20251020202607 → 0.0.0-execa-dynamic-import-20260304221256
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +847 -6
- package/LICENSE.md +15 -0
- package/docs/INVESTIGATION_WORKFLOW.md +314 -0
- package/package.json +34 -26
- package/scripts/download.ts +22 -1
- package/scripts/generate-wm-templates.ts +2 -1
- package/src/cli.ts +967 -155
- package/src/commands/__tests__/prepare.test.ts +11 -6
- package/src/commands/__tests__/run.test.ts +10 -8
- package/src/commands/clean.ts +147 -0
- package/src/commands/deterministic-ids.ts +126 -0
- package/src/commands/find-prohibited.ts +299 -0
- package/src/commands/investigate.ts +2437 -0
- package/src/commands/list-partial.ts +137 -0
- package/src/commands/obscure-thread-ids.ts +237 -0
- package/src/commands/partial-results.ts +198 -0
- package/src/commands/precompute-embeddings.ts +302 -0
- package/src/commands/prepare.ts +732 -105
- package/src/commands/reconcile.ts +383 -0
- package/src/commands/run.ts +1240 -121
- package/src/commands/sessions.ts +126 -0
- package/src/commands/sync.ts +152 -0
- package/src/commands/test-cleaned.ts +53 -0
- package/src/commands/test-incremental.ts +120 -0
- package/src/commands/test-messages.ts +95 -0
- package/src/commands/test-prohibited.ts +244 -0
- package/src/commands/test-thread.ts +136 -0
- package/src/commands/tokens.ts +625 -0
- package/src/config.ts +1253 -72
- package/src/data/types.ts +48 -6
- package/src/embeddings/cached-openai-embedding-model.ts +0 -1
- package/src/evaluation/longmemeval-metric.ts +186 -103
- package/src/processors/content-sanitizer.ts +153 -0
- package/src/processors/date-injector.ts +111 -0
- package/src/processors/index.ts +3 -0
- package/src/processors/observation-semantic-filter.ts +1142 -0
- package/src/retry-model.ts +5 -3
- package/src/storage/__tests__/benchmark-store.test.ts +18 -35
- package/src/storage/benchmark-store.ts +140 -153
- package/src/storage/benchmark-vector.ts +1 -1
- package/src/storage/index.ts +1 -0
- package/src/storage/persistable-inmemory.ts +133 -0
- package/src/test-utils/mock-model.ts +35 -0
|
@@ -0,0 +1,625 @@
|
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import ora from 'ora';
|
|
3
|
+
import { join } from 'path';
|
|
4
|
+
import { readFile } from 'fs/promises';
|
|
5
|
+
import { existsSync } from 'fs';
|
|
6
|
+
|
|
7
|
+
import { DatasetLoader } from '../data/loader';
|
|
8
|
+
import type { DatasetType, LongMemEvalQuestion } from '../data/types';
|
|
9
|
+
import { TokenCounter } from '@mastra/memory/processors';
|
|
10
|
+
|
|
11
|
+
export interface TokensOptions {
|
|
12
|
+
dataset: DatasetType;
|
|
13
|
+
questionId?: string;
|
|
14
|
+
offset?: number;
|
|
15
|
+
subset?: number;
|
|
16
|
+
preparedDataDir?: string;
|
|
17
|
+
showSessions?: boolean;
|
|
18
|
+
topN?: number;
|
|
19
|
+
observationsOnly?: boolean;
|
|
20
|
+
config?: string;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
interface SessionStats {
|
|
24
|
+
sessionId: string;
|
|
25
|
+
isAnswerSession: boolean;
|
|
26
|
+
tokens: number;
|
|
27
|
+
messageCount: number;
|
|
28
|
+
userTokens: number;
|
|
29
|
+
assistantTokens: number;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
interface QuestionStats {
|
|
33
|
+
questionId: string;
|
|
34
|
+
questionType: string;
|
|
35
|
+
totalTokens: number;
|
|
36
|
+
answerSessionTokens: number;
|
|
37
|
+
messageCount: number;
|
|
38
|
+
sessionCount: number;
|
|
39
|
+
answerSessionCount: number;
|
|
40
|
+
largestSession: { id: string; tokens: number };
|
|
41
|
+
smallestSession: { id: string; tokens: number };
|
|
42
|
+
avgTokensPerMessage: number;
|
|
43
|
+
sessions: SessionStats[];
|
|
44
|
+
// Prepared data comparison (if available)
|
|
45
|
+
preparedObservationTokens?: number;
|
|
46
|
+
compressionRatio?: number;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
interface ObservationOnlyStats {
|
|
50
|
+
questionId: string;
|
|
51
|
+
questionType: string;
|
|
52
|
+
observationTokens: number;
|
|
53
|
+
found: boolean;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
interface AggregateStats {
|
|
57
|
+
totalQuestions: number;
|
|
58
|
+
totalTokens: number;
|
|
59
|
+
totalMessages: number;
|
|
60
|
+
avgTokensPerQuestion: number;
|
|
61
|
+
minTokens: number;
|
|
62
|
+
maxTokens: number;
|
|
63
|
+
medianTokens: number;
|
|
64
|
+
p90Tokens: number;
|
|
65
|
+
p95Tokens: number;
|
|
66
|
+
distribution: { range: string; count: number }[];
|
|
67
|
+
topQuestions: { questionId: string; tokens: number }[];
|
|
68
|
+
estimatedCost: {
|
|
69
|
+
gpt4o: number;
|
|
70
|
+
gpt4oMini: number;
|
|
71
|
+
claude35Sonnet: number;
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export class TokensCommand {
|
|
76
|
+
private loader: DatasetLoader;
|
|
77
|
+
private tokenCounter: TokenCounter;
|
|
78
|
+
private preparedDataDir: string;
|
|
79
|
+
|
|
80
|
+
constructor() {
|
|
81
|
+
this.loader = new DatasetLoader();
|
|
82
|
+
this.tokenCounter = new TokenCounter();
|
|
83
|
+
this.preparedDataDir = './prepared-data';
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async run(options: TokensOptions): Promise<void> {
|
|
87
|
+
// Fast path for observations-only mode
|
|
88
|
+
if (options.observationsOnly) {
|
|
89
|
+
await this.runObservationsOnly(options);
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const spinner = ora('Loading dataset...').start();
|
|
94
|
+
|
|
95
|
+
try {
|
|
96
|
+
// Load dataset
|
|
97
|
+
const questions = await this.loader.loadDataset(options.dataset);
|
|
98
|
+
spinner.succeed(`Loaded ${questions.length} questions`);
|
|
99
|
+
|
|
100
|
+
// Filter questions based on options
|
|
101
|
+
let selectedQuestions: LongMemEvalQuestion[];
|
|
102
|
+
|
|
103
|
+
if (options.questionId) {
|
|
104
|
+
const question = questions.find(q => q.question_id === options.questionId);
|
|
105
|
+
if (!question) {
|
|
106
|
+
console.log(chalk.red(`Question ${options.questionId} not found`));
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
selectedQuestions = [question];
|
|
110
|
+
} else {
|
|
111
|
+
const offset = options.offset || 0;
|
|
112
|
+
selectedQuestions = questions.slice(offset);
|
|
113
|
+
if (options.subset) {
|
|
114
|
+
selectedQuestions = selectedQuestions.slice(0, options.subset);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
console.log(chalk.blue(`\n📊 Token Estimate for ${selectedQuestions.length} question(s)\n`));
|
|
119
|
+
|
|
120
|
+
// Calculate stats for each question
|
|
121
|
+
const questionStats: QuestionStats[] = [];
|
|
122
|
+
const statsSpinner = ora('Calculating token counts...').start();
|
|
123
|
+
|
|
124
|
+
for (let i = 0; i < selectedQuestions.length; i++) {
|
|
125
|
+
const question = selectedQuestions[i];
|
|
126
|
+
statsSpinner.text = `Calculating token counts... ${i + 1}/${selectedQuestions.length}`;
|
|
127
|
+
|
|
128
|
+
const stats = await this.calculateQuestionStats(question, options);
|
|
129
|
+
questionStats.push(stats);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
statsSpinner.succeed('Token counts calculated');
|
|
133
|
+
|
|
134
|
+
// Display results
|
|
135
|
+
if (selectedQuestions.length === 1) {
|
|
136
|
+
this.displaySingleQuestion(questionStats[0], options.showSessions);
|
|
137
|
+
} else {
|
|
138
|
+
const aggregate = this.calculateAggregateStats(questionStats, options.topN || 10);
|
|
139
|
+
this.displayAggregateStats(aggregate, questionStats, options.showSessions);
|
|
140
|
+
}
|
|
141
|
+
} catch (error) {
|
|
142
|
+
spinner.fail('Error loading dataset');
|
|
143
|
+
throw error;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Fast path: only count observation tokens from prepared data (skip raw session analysis)
|
|
149
|
+
*/
|
|
150
|
+
private async runObservationsOnly(options: TokensOptions): Promise<void> {
|
|
151
|
+
const config = options.config || 'observational-memory';
|
|
152
|
+
const preparedDir = options.preparedDataDir || this.preparedDataDir;
|
|
153
|
+
const configDir = join(preparedDir, options.dataset, config);
|
|
154
|
+
|
|
155
|
+
// Check if config directory exists
|
|
156
|
+
if (!existsSync(configDir)) {
|
|
157
|
+
console.log(chalk.red(`Config directory not found: ${configDir}`));
|
|
158
|
+
console.log(chalk.gray(`Use --config to specify a different config, or prepare data first.`));
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
const spinner = ora('Scanning prepared data...').start();
|
|
163
|
+
|
|
164
|
+
// Get all question directories
|
|
165
|
+
const { readdirSync } = await import('fs');
|
|
166
|
+
const questionDirs = readdirSync(configDir, { withFileTypes: true })
|
|
167
|
+
.filter(d => d.isDirectory())
|
|
168
|
+
.map(d => d.name);
|
|
169
|
+
|
|
170
|
+
// Filter by questionId if specified
|
|
171
|
+
let targetQuestions = questionDirs;
|
|
172
|
+
if (options.questionId) {
|
|
173
|
+
if (!questionDirs.includes(options.questionId)) {
|
|
174
|
+
spinner.fail(`Question ${options.questionId} not found in ${config}`);
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
targetQuestions = [options.questionId];
|
|
178
|
+
} else {
|
|
179
|
+
// Apply offset and subset
|
|
180
|
+
const offset = options.offset || 0;
|
|
181
|
+
targetQuestions = questionDirs.slice(offset);
|
|
182
|
+
if (options.subset) {
|
|
183
|
+
targetQuestions = targetQuestions.slice(0, options.subset);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
spinner.text = `Counting observation tokens for ${targetQuestions.length} questions...`;
|
|
188
|
+
|
|
189
|
+
const stats: ObservationOnlyStats[] = [];
|
|
190
|
+
let totalTokens = 0;
|
|
191
|
+
let foundCount = 0;
|
|
192
|
+
|
|
193
|
+
for (const questionId of targetQuestions) {
|
|
194
|
+
const omJsonPath = join(configDir, questionId, 'om.json');
|
|
195
|
+
let observationTokens = 0;
|
|
196
|
+
let found = false;
|
|
197
|
+
|
|
198
|
+
if (existsSync(omJsonPath)) {
|
|
199
|
+
try {
|
|
200
|
+
const omData = JSON.parse(await readFile(omJsonPath, 'utf-8'));
|
|
201
|
+
|
|
202
|
+
// Sum activeObservations across ALL records
|
|
203
|
+
if (Array.isArray(omData.observationalMemory) && omData.observationalMemory[0]) {
|
|
204
|
+
const [, records] = omData.observationalMemory[0];
|
|
205
|
+
if (Array.isArray(records)) {
|
|
206
|
+
for (const record of records) {
|
|
207
|
+
if (record?.activeObservations) {
|
|
208
|
+
observationTokens += this.tokenCounter.countString(record.activeObservations);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
} else if (omData.record?.activeObservations) {
|
|
213
|
+
observationTokens = this.tokenCounter.countString(omData.record.activeObservations);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
if (observationTokens > 0) {
|
|
217
|
+
found = true;
|
|
218
|
+
foundCount++;
|
|
219
|
+
totalTokens += observationTokens;
|
|
220
|
+
}
|
|
221
|
+
} catch {
|
|
222
|
+
// Ignore parse errors
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// Try to get question type from meta.json
|
|
227
|
+
let questionType = 'unknown';
|
|
228
|
+
const metaPath = join(configDir, questionId, 'meta.json');
|
|
229
|
+
if (existsSync(metaPath)) {
|
|
230
|
+
try {
|
|
231
|
+
const meta = JSON.parse(await readFile(metaPath, 'utf-8'));
|
|
232
|
+
questionType = meta.questionType || 'unknown';
|
|
233
|
+
} catch {
|
|
234
|
+
// Ignore
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
stats.push({ questionId, questionType, observationTokens, found });
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
spinner.succeed(`Counted ${foundCount}/${targetQuestions.length} questions with observations`);
|
|
242
|
+
|
|
243
|
+
// Display results
|
|
244
|
+
console.log(chalk.blue(`\n📊 Observation Tokens (${config})\n`));
|
|
245
|
+
|
|
246
|
+
if (targetQuestions.length === 1) {
|
|
247
|
+
// Single question view
|
|
248
|
+
const s = stats[0];
|
|
249
|
+
console.log(chalk.bold(`Question: ${s.questionId}`));
|
|
250
|
+
console.log(chalk.gray(`Type: ${s.questionType}`));
|
|
251
|
+
console.log();
|
|
252
|
+
if (s.found) {
|
|
253
|
+
console.log(` Observation tokens: ${this.formatTokens(s.observationTokens)}`);
|
|
254
|
+
} else {
|
|
255
|
+
console.log(chalk.yellow(` No observations found`));
|
|
256
|
+
}
|
|
257
|
+
} else {
|
|
258
|
+
// Aggregate view
|
|
259
|
+
const tokenCounts = stats
|
|
260
|
+
.filter(s => s.found)
|
|
261
|
+
.map(s => s.observationTokens)
|
|
262
|
+
.sort((a, b) => a - b);
|
|
263
|
+
|
|
264
|
+
const percentile = (arr: number[], p: number) => {
|
|
265
|
+
if (arr.length === 0) return 0;
|
|
266
|
+
const idx = Math.ceil((p / 100) * arr.length) - 1;
|
|
267
|
+
return arr[Math.max(0, idx)];
|
|
268
|
+
};
|
|
269
|
+
|
|
270
|
+
console.log(chalk.bold('Summary:'));
|
|
271
|
+
console.log(` Questions with data: ${foundCount}/${targetQuestions.length}`);
|
|
272
|
+
console.log(` Total tokens: ${this.formatTokens(totalTokens)}`);
|
|
273
|
+
console.log(
|
|
274
|
+
` Avg tokens/question: ${this.formatTokens(foundCount > 0 ? Math.round(totalTokens / foundCount) : 0)}`,
|
|
275
|
+
);
|
|
276
|
+
console.log();
|
|
277
|
+
|
|
278
|
+
if (tokenCounts.length > 0) {
|
|
279
|
+
console.log(chalk.bold('Distribution:'));
|
|
280
|
+
console.log(` Min: ${this.formatTokens(tokenCounts[0])}`);
|
|
281
|
+
console.log(` Median: ${this.formatTokens(percentile(tokenCounts, 50))}`);
|
|
282
|
+
console.log(` P90: ${this.formatTokens(percentile(tokenCounts, 90))}`);
|
|
283
|
+
console.log(` P95: ${this.formatTokens(percentile(tokenCounts, 95))}`);
|
|
284
|
+
console.log(` Max: ${this.formatTokens(tokenCounts[tokenCounts.length - 1])}`);
|
|
285
|
+
console.log();
|
|
286
|
+
|
|
287
|
+
// Top N largest
|
|
288
|
+
const topN = options.topN || 10;
|
|
289
|
+
const sorted = [...stats].filter(s => s.found).sort((a, b) => b.observationTokens - a.observationTokens);
|
|
290
|
+
console.log(chalk.bold(`Top ${Math.min(topN, sorted.length)} Largest:`));
|
|
291
|
+
for (let i = 0; i < Math.min(topN, sorted.length); i++) {
|
|
292
|
+
const s = sorted[i];
|
|
293
|
+
console.log(
|
|
294
|
+
` ${(i + 1).toString().padStart(2)}. ${s.questionId} ${this.formatTokens(s.observationTokens)}`,
|
|
295
|
+
);
|
|
296
|
+
}
|
|
297
|
+
console.log();
|
|
298
|
+
|
|
299
|
+
// Cost estimates
|
|
300
|
+
const gpt4oInputPrice = 2.5;
|
|
301
|
+
const gpt4oMiniInputPrice = 0.15;
|
|
302
|
+
console.log(chalk.bold('Estimated Input Costs:'));
|
|
303
|
+
console.log(` GPT-4o: $${((totalTokens / 1_000_000) * gpt4oInputPrice).toFixed(2)}`);
|
|
304
|
+
console.log(` GPT-4o-mini: $${((totalTokens / 1_000_000) * gpt4oMiniInputPrice).toFixed(2)}`);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
console.log();
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
private async calculateQuestionStats(question: LongMemEvalQuestion, options: TokensOptions): Promise<QuestionStats> {
|
|
312
|
+
const sessions: SessionStats[] = [];
|
|
313
|
+
const answerSessionIdSet = new Set(question.answer_session_ids || []);
|
|
314
|
+
|
|
315
|
+
// haystack_sessions is Turn[][] and haystack_session_ids is string[]
|
|
316
|
+
// They are parallel arrays
|
|
317
|
+
for (let i = 0; i < question.haystack_sessions.length; i++) {
|
|
318
|
+
const sessionId = question.haystack_session_ids[i];
|
|
319
|
+
const messages = question.haystack_sessions[i];
|
|
320
|
+
const isAnswerSession = answerSessionIdSet.has(sessionId);
|
|
321
|
+
|
|
322
|
+
// Count tokens in messages
|
|
323
|
+
let tokens = 0;
|
|
324
|
+
let userTokens = 0;
|
|
325
|
+
let assistantTokens = 0;
|
|
326
|
+
let messageCount = 0;
|
|
327
|
+
|
|
328
|
+
for (const msg of messages) {
|
|
329
|
+
// Count tokens directly from the string content
|
|
330
|
+
// Add role overhead (~4 tokens) to match TokenCounter.countMessage behavior
|
|
331
|
+
const contentTokens = this.tokenCounter.countString(msg.content);
|
|
332
|
+
const roleTokens = this.tokenCounter.countString(msg.role);
|
|
333
|
+
const msgTokens = contentTokens + roleTokens + 4; // ~4 tokens overhead per message
|
|
334
|
+
|
|
335
|
+
tokens += msgTokens;
|
|
336
|
+
messageCount++;
|
|
337
|
+
|
|
338
|
+
if (msg.role === 'user') {
|
|
339
|
+
userTokens += msgTokens;
|
|
340
|
+
} else {
|
|
341
|
+
assistantTokens += msgTokens;
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
sessions.push({
|
|
346
|
+
sessionId,
|
|
347
|
+
isAnswerSession,
|
|
348
|
+
tokens,
|
|
349
|
+
messageCount,
|
|
350
|
+
userTokens,
|
|
351
|
+
assistantTokens,
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
// Calculate aggregates
|
|
356
|
+
const totalTokens = sessions.reduce((sum, s) => sum + s.tokens, 0);
|
|
357
|
+
const answerSessionTokens = sessions.filter(s => s.isAnswerSession).reduce((sum, s) => sum + s.tokens, 0);
|
|
358
|
+
const messageCount = sessions.reduce((sum, s) => sum + s.messageCount, 0);
|
|
359
|
+
|
|
360
|
+
const sortedByTokens = [...sessions].sort((a, b) => b.tokens - a.tokens);
|
|
361
|
+
const largestSession = sortedByTokens[0] || { id: '', tokens: 0 };
|
|
362
|
+
const smallestSession = sortedByTokens[sortedByTokens.length - 1] || { id: '', tokens: 0 };
|
|
363
|
+
|
|
364
|
+
// Check for prepared data
|
|
365
|
+
let preparedObservationTokens: number | undefined;
|
|
366
|
+
let compressionRatio: number | undefined;
|
|
367
|
+
|
|
368
|
+
const preparedDir = options.preparedDataDir || this.preparedDataDir;
|
|
369
|
+
// Check all observational-memory config directories
|
|
370
|
+
const omConfigs = ['observational-memory', 'observational-memory-shortcut', 'observational-memory-shortcut-glm'];
|
|
371
|
+
let omJsonPath = '';
|
|
372
|
+
for (const config of omConfigs) {
|
|
373
|
+
const candidatePath = join(preparedDir, options.dataset, config, question.question_id, 'om.json');
|
|
374
|
+
if (existsSync(candidatePath)) {
|
|
375
|
+
omJsonPath = candidatePath;
|
|
376
|
+
break;
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
if (omJsonPath && existsSync(omJsonPath)) {
|
|
381
|
+
try {
|
|
382
|
+
const omData = JSON.parse(await readFile(omJsonPath, 'utf-8'));
|
|
383
|
+
// The structure is: observationalMemory[0] = [resourceKey, [records...]]
|
|
384
|
+
// Sum activeObservations across ALL records (each reflection creates a new record)
|
|
385
|
+
let totalObservationTokens = 0;
|
|
386
|
+
|
|
387
|
+
if (Array.isArray(omData.observationalMemory) && omData.observationalMemory[0]) {
|
|
388
|
+
const [, records] = omData.observationalMemory[0];
|
|
389
|
+
if (Array.isArray(records)) {
|
|
390
|
+
for (const record of records) {
|
|
391
|
+
if (record?.activeObservations) {
|
|
392
|
+
totalObservationTokens += this.tokenCounter.countString(record.activeObservations);
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
} else if (omData.record?.activeObservations) {
|
|
397
|
+
// Fallback for old structure
|
|
398
|
+
totalObservationTokens = this.tokenCounter.countString(omData.record.activeObservations);
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
if (totalObservationTokens > 0) {
|
|
402
|
+
preparedObservationTokens = totalObservationTokens;
|
|
403
|
+
if (totalTokens > 0) {
|
|
404
|
+
compressionRatio = totalTokens / preparedObservationTokens;
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
} catch {
|
|
408
|
+
// Ignore errors reading prepared data
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
return {
|
|
413
|
+
questionId: question.question_id,
|
|
414
|
+
questionType: question.question_type,
|
|
415
|
+
totalTokens,
|
|
416
|
+
answerSessionTokens,
|
|
417
|
+
messageCount,
|
|
418
|
+
sessionCount: sessions.length,
|
|
419
|
+
answerSessionCount: sessions.filter(s => s.isAnswerSession).length,
|
|
420
|
+
largestSession: { id: largestSession.sessionId, tokens: largestSession.tokens },
|
|
421
|
+
smallestSession: { id: smallestSession.sessionId, tokens: smallestSession.tokens },
|
|
422
|
+
avgTokensPerMessage: messageCount > 0 ? Math.round(totalTokens / messageCount) : 0,
|
|
423
|
+
sessions,
|
|
424
|
+
preparedObservationTokens,
|
|
425
|
+
compressionRatio,
|
|
426
|
+
};
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
private calculateAggregateStats(questionStats: QuestionStats[], topN: number): AggregateStats {
|
|
430
|
+
const tokenCounts = questionStats.map(q => q.totalTokens).sort((a, b) => a - b);
|
|
431
|
+
const totalTokens = tokenCounts.reduce((sum, t) => sum + t, 0);
|
|
432
|
+
const totalMessages = questionStats.reduce((sum, q) => sum + q.messageCount, 0);
|
|
433
|
+
|
|
434
|
+
// Percentiles
|
|
435
|
+
const percentile = (arr: number[], p: number) => {
|
|
436
|
+
const idx = Math.ceil((p / 100) * arr.length) - 1;
|
|
437
|
+
return arr[Math.max(0, idx)];
|
|
438
|
+
};
|
|
439
|
+
|
|
440
|
+
// Distribution buckets
|
|
441
|
+
const buckets = [
|
|
442
|
+
{ min: 0, max: 10000, label: '0-10k' },
|
|
443
|
+
{ min: 10000, max: 25000, label: '10k-25k' },
|
|
444
|
+
{ min: 25000, max: 50000, label: '25k-50k' },
|
|
445
|
+
{ min: 50000, max: 100000, label: '50k-100k' },
|
|
446
|
+
{ min: 100000, max: 250000, label: '100k-250k' },
|
|
447
|
+
{ min: 250000, max: Infinity, label: '250k+' },
|
|
448
|
+
];
|
|
449
|
+
|
|
450
|
+
const distribution = buckets.map(bucket => ({
|
|
451
|
+
range: bucket.label,
|
|
452
|
+
count: questionStats.filter(q => q.totalTokens >= bucket.min && q.totalTokens < bucket.max).length,
|
|
453
|
+
}));
|
|
454
|
+
|
|
455
|
+
// Top N largest questions
|
|
456
|
+
const topQuestions = [...questionStats]
|
|
457
|
+
.sort((a, b) => b.totalTokens - a.totalTokens)
|
|
458
|
+
.slice(0, topN)
|
|
459
|
+
.map(q => ({ questionId: q.questionId, tokens: q.totalTokens }));
|
|
460
|
+
|
|
461
|
+
// Cost estimates (per 1M tokens)
|
|
462
|
+
const gpt4oInputPrice = 2.5; // $2.50 per 1M input tokens
|
|
463
|
+
const gpt4oMiniInputPrice = 0.15; // $0.15 per 1M input tokens
|
|
464
|
+
const claude35SonnetInputPrice = 3.0; // $3.00 per 1M input tokens
|
|
465
|
+
|
|
466
|
+
return {
|
|
467
|
+
totalQuestions: questionStats.length,
|
|
468
|
+
totalTokens,
|
|
469
|
+
totalMessages,
|
|
470
|
+
avgTokensPerQuestion: Math.round(totalTokens / questionStats.length),
|
|
471
|
+
minTokens: tokenCounts[0] || 0,
|
|
472
|
+
maxTokens: tokenCounts[tokenCounts.length - 1] || 0,
|
|
473
|
+
medianTokens: percentile(tokenCounts, 50),
|
|
474
|
+
p90Tokens: percentile(tokenCounts, 90),
|
|
475
|
+
p95Tokens: percentile(tokenCounts, 95),
|
|
476
|
+
distribution,
|
|
477
|
+
topQuestions,
|
|
478
|
+
estimatedCost: {
|
|
479
|
+
gpt4o: (totalTokens / 1_000_000) * gpt4oInputPrice,
|
|
480
|
+
gpt4oMini: (totalTokens / 1_000_000) * gpt4oMiniInputPrice,
|
|
481
|
+
claude35Sonnet: (totalTokens / 1_000_000) * claude35SonnetInputPrice,
|
|
482
|
+
},
|
|
483
|
+
};
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
private displaySingleQuestion(stats: QuestionStats, showSessions?: boolean): void {
|
|
487
|
+
console.log(chalk.bold(`Question: ${stats.questionId}`));
|
|
488
|
+
console.log(chalk.gray(`Type: ${stats.questionType}`));
|
|
489
|
+
console.log();
|
|
490
|
+
|
|
491
|
+
if (showSessions) {
|
|
492
|
+
console.log(chalk.bold(`Sessions (${stats.sessionCount} total):\n`));
|
|
493
|
+
|
|
494
|
+
// Sort sessions by tokens descending
|
|
495
|
+
const sortedSessions = [...stats.sessions].sort((a, b) => b.tokens - a.tokens);
|
|
496
|
+
|
|
497
|
+
for (const session of sortedSessions) {
|
|
498
|
+
const answerTag = session.isAnswerSession ? chalk.green(' (answer)') : '';
|
|
499
|
+
const tokens = this.formatTokens(session.tokens);
|
|
500
|
+
const breakdown = chalk.gray(
|
|
501
|
+
`(${session.messageCount} msgs, U:${this.formatTokens(session.userTokens)} A:${this.formatTokens(session.assistantTokens)})`,
|
|
502
|
+
);
|
|
503
|
+
|
|
504
|
+
console.log(` ${session.sessionId}${answerTag}`);
|
|
505
|
+
console.log(` ${tokens} ${breakdown}`);
|
|
506
|
+
}
|
|
507
|
+
console.log();
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
console.log(chalk.bold('Summary:'));
|
|
511
|
+
console.log(` Total tokens: ${this.formatTokens(stats.totalTokens)}`);
|
|
512
|
+
console.log(
|
|
513
|
+
` Answer session tokens: ${this.formatTokens(stats.answerSessionTokens)} ${chalk.gray(`(${stats.answerSessionCount} sessions)`)}`,
|
|
514
|
+
);
|
|
515
|
+
console.log(` Total messages: ${stats.messageCount}`);
|
|
516
|
+
console.log(` Avg tokens/message: ${stats.avgTokensPerMessage}`);
|
|
517
|
+
console.log(
|
|
518
|
+
` Largest session: ${stats.largestSession.id} (${this.formatTokens(stats.largestSession.tokens)})`,
|
|
519
|
+
);
|
|
520
|
+
console.log(
|
|
521
|
+
` Smallest session: ${stats.smallestSession.id} (${this.formatTokens(stats.smallestSession.tokens)})`,
|
|
522
|
+
);
|
|
523
|
+
|
|
524
|
+
if (stats.preparedObservationTokens !== undefined) {
|
|
525
|
+
console.log();
|
|
526
|
+
console.log(chalk.bold('Prepared Data (OM):'));
|
|
527
|
+
console.log(` Observation tokens: ${this.formatTokens(stats.preparedObservationTokens)}`);
|
|
528
|
+
if (stats.compressionRatio !== undefined) {
|
|
529
|
+
console.log(` Compression ratio: ${stats.compressionRatio.toFixed(1)}x`);
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
console.log();
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
private displayAggregateStats(
|
|
537
|
+
aggregate: AggregateStats,
|
|
538
|
+
questionStats: QuestionStats[],
|
|
539
|
+
showSessions?: boolean,
|
|
540
|
+
): void {
|
|
541
|
+
// Summary
|
|
542
|
+
console.log(chalk.bold('Summary:'));
|
|
543
|
+
console.log(` Questions: ${aggregate.totalQuestions}`);
|
|
544
|
+
console.log(` Total tokens: ${this.formatTokens(aggregate.totalTokens)}`);
|
|
545
|
+
console.log(` Total messages: ${aggregate.totalMessages.toLocaleString()}`);
|
|
546
|
+
console.log(` Avg tokens/question: ${this.formatTokens(aggregate.avgTokensPerQuestion)}`);
|
|
547
|
+
console.log();
|
|
548
|
+
|
|
549
|
+
// Distribution
|
|
550
|
+
console.log(chalk.bold('Token Distribution:'));
|
|
551
|
+
console.log(` Min: ${this.formatTokens(aggregate.minTokens)}`);
|
|
552
|
+
console.log(` Median: ${this.formatTokens(aggregate.medianTokens)}`);
|
|
553
|
+
console.log(` P90: ${this.formatTokens(aggregate.p90Tokens)}`);
|
|
554
|
+
console.log(` P95: ${this.formatTokens(aggregate.p95Tokens)}`);
|
|
555
|
+
console.log(` Max: ${this.formatTokens(aggregate.maxTokens)}`);
|
|
556
|
+
console.log();
|
|
557
|
+
|
|
558
|
+
// Histogram
|
|
559
|
+
console.log(chalk.bold('Distribution by Range:'));
|
|
560
|
+
const maxCount = Math.max(...aggregate.distribution.map(d => d.count));
|
|
561
|
+
const barWidth = 30;
|
|
562
|
+
|
|
563
|
+
for (const bucket of aggregate.distribution) {
|
|
564
|
+
if (bucket.count === 0) continue;
|
|
565
|
+
const barLength = Math.round((bucket.count / maxCount) * barWidth);
|
|
566
|
+
const bar = '█'.repeat(barLength);
|
|
567
|
+
const pct = ((bucket.count / aggregate.totalQuestions) * 100).toFixed(0);
|
|
568
|
+
console.log(` ${bucket.range.padEnd(10)} ${bar} ${bucket.count} (${pct}%)`);
|
|
569
|
+
}
|
|
570
|
+
console.log();
|
|
571
|
+
|
|
572
|
+
// Top N largest
|
|
573
|
+
console.log(chalk.bold(`Top ${aggregate.topQuestions.length} Largest Questions:`));
|
|
574
|
+
for (let i = 0; i < aggregate.topQuestions.length; i++) {
|
|
575
|
+
const q = aggregate.topQuestions[i];
|
|
576
|
+
console.log(` ${(i + 1).toString().padStart(2)}. ${q.questionId} ${this.formatTokens(q.tokens)}`);
|
|
577
|
+
}
|
|
578
|
+
console.log();
|
|
579
|
+
|
|
580
|
+
// Cost estimates
|
|
581
|
+
console.log(chalk.bold('Estimated Input Costs:'));
|
|
582
|
+
console.log(` GPT-4o: $${aggregate.estimatedCost.gpt4o.toFixed(2)}`);
|
|
583
|
+
console.log(` GPT-4o-mini: $${aggregate.estimatedCost.gpt4oMini.toFixed(2)}`);
|
|
584
|
+
console.log(` Claude 3.5 Sonnet: $${aggregate.estimatedCost.claude35Sonnet.toFixed(2)}`);
|
|
585
|
+
console.log(chalk.gray(' (Input tokens only, excludes output/observation costs)'));
|
|
586
|
+
console.log();
|
|
587
|
+
|
|
588
|
+
// Compression stats if any have prepared data
|
|
589
|
+
const withPrepared = questionStats.filter(q => q.preparedObservationTokens !== undefined);
|
|
590
|
+
if (withPrepared.length > 0) {
|
|
591
|
+
const totalPreparedTokens = withPrepared.reduce((sum, q) => sum + (q.preparedObservationTokens || 0), 0);
|
|
592
|
+
const totalOriginalTokens = withPrepared.reduce((sum, q) => sum + q.totalTokens, 0);
|
|
593
|
+
const avgCompression = totalOriginalTokens / totalPreparedTokens;
|
|
594
|
+
|
|
595
|
+
console.log(chalk.bold('Prepared Data (OM):'));
|
|
596
|
+
console.log(` Questions with OM: ${withPrepared.length}`);
|
|
597
|
+
console.log(` Total OM tokens: ${this.formatTokens(totalPreparedTokens)}`);
|
|
598
|
+
console.log(` Avg compression: ${avgCompression.toFixed(1)}x`);
|
|
599
|
+
console.log();
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
// Show per-question details if requested
|
|
603
|
+
if (showSessions) {
|
|
604
|
+
console.log(chalk.bold('Per-Question Details:\n'));
|
|
605
|
+
for (const stats of questionStats) {
|
|
606
|
+
const compression = stats.compressionRatio
|
|
607
|
+
? chalk.green(` (${stats.compressionRatio.toFixed(1)}x compression)`)
|
|
608
|
+
: '';
|
|
609
|
+
console.log(
|
|
610
|
+
` ${stats.questionId} - ${this.formatTokens(stats.totalTokens)} (${stats.sessionCount} sessions, ${stats.messageCount} msgs)${compression}`,
|
|
611
|
+
);
|
|
612
|
+
}
|
|
613
|
+
console.log();
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
private formatTokens(tokens: number): string {
|
|
618
|
+
if (tokens >= 1_000_000) {
|
|
619
|
+
return `${(tokens / 1_000_000).toFixed(2)}M`;
|
|
620
|
+
} else if (tokens >= 1_000) {
|
|
621
|
+
return `${(tokens / 1_000).toFixed(1)}k`;
|
|
622
|
+
}
|
|
623
|
+
return tokens.toString();
|
|
624
|
+
}
|
|
625
|
+
}
|