@mastra/longmemeval 0.0.0-error-handler-fix-20251020202607 → 0.0.0-execa-dynamic-import-20260304221256

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/CHANGELOG.md +847 -6
  2. package/LICENSE.md +15 -0
  3. package/docs/INVESTIGATION_WORKFLOW.md +314 -0
  4. package/package.json +34 -26
  5. package/scripts/download.ts +22 -1
  6. package/scripts/generate-wm-templates.ts +2 -1
  7. package/src/cli.ts +967 -155
  8. package/src/commands/__tests__/prepare.test.ts +11 -6
  9. package/src/commands/__tests__/run.test.ts +10 -8
  10. package/src/commands/clean.ts +147 -0
  11. package/src/commands/deterministic-ids.ts +126 -0
  12. package/src/commands/find-prohibited.ts +299 -0
  13. package/src/commands/investigate.ts +2437 -0
  14. package/src/commands/list-partial.ts +137 -0
  15. package/src/commands/obscure-thread-ids.ts +237 -0
  16. package/src/commands/partial-results.ts +198 -0
  17. package/src/commands/precompute-embeddings.ts +302 -0
  18. package/src/commands/prepare.ts +732 -105
  19. package/src/commands/reconcile.ts +383 -0
  20. package/src/commands/run.ts +1240 -121
  21. package/src/commands/sessions.ts +126 -0
  22. package/src/commands/sync.ts +152 -0
  23. package/src/commands/test-cleaned.ts +53 -0
  24. package/src/commands/test-incremental.ts +120 -0
  25. package/src/commands/test-messages.ts +95 -0
  26. package/src/commands/test-prohibited.ts +244 -0
  27. package/src/commands/test-thread.ts +136 -0
  28. package/src/commands/tokens.ts +625 -0
  29. package/src/config.ts +1253 -72
  30. package/src/data/types.ts +48 -6
  31. package/src/embeddings/cached-openai-embedding-model.ts +0 -1
  32. package/src/evaluation/longmemeval-metric.ts +186 -103
  33. package/src/processors/content-sanitizer.ts +153 -0
  34. package/src/processors/date-injector.ts +111 -0
  35. package/src/processors/index.ts +3 -0
  36. package/src/processors/observation-semantic-filter.ts +1142 -0
  37. package/src/retry-model.ts +5 -3
  38. package/src/storage/__tests__/benchmark-store.test.ts +18 -35
  39. package/src/storage/benchmark-store.ts +140 -153
  40. package/src/storage/benchmark-vector.ts +1 -1
  41. package/src/storage/index.ts +1 -0
  42. package/src/storage/persistable-inmemory.ts +133 -0
  43. package/src/test-utils/mock-model.ts +35 -0
@@ -0,0 +1,625 @@
1
+ import chalk from 'chalk';
2
+ import ora from 'ora';
3
+ import { join } from 'path';
4
+ import { readFile } from 'fs/promises';
5
+ import { existsSync } from 'fs';
6
+
7
+ import { DatasetLoader } from '../data/loader';
8
+ import type { DatasetType, LongMemEvalQuestion } from '../data/types';
9
+ import { TokenCounter } from '@mastra/memory/processors';
10
+
11
+ export interface TokensOptions {
12
+ dataset: DatasetType;
13
+ questionId?: string;
14
+ offset?: number;
15
+ subset?: number;
16
+ preparedDataDir?: string;
17
+ showSessions?: boolean;
18
+ topN?: number;
19
+ observationsOnly?: boolean;
20
+ config?: string;
21
+ }
22
+
23
+ interface SessionStats {
24
+ sessionId: string;
25
+ isAnswerSession: boolean;
26
+ tokens: number;
27
+ messageCount: number;
28
+ userTokens: number;
29
+ assistantTokens: number;
30
+ }
31
+
32
+ interface QuestionStats {
33
+ questionId: string;
34
+ questionType: string;
35
+ totalTokens: number;
36
+ answerSessionTokens: number;
37
+ messageCount: number;
38
+ sessionCount: number;
39
+ answerSessionCount: number;
40
+ largestSession: { id: string; tokens: number };
41
+ smallestSession: { id: string; tokens: number };
42
+ avgTokensPerMessage: number;
43
+ sessions: SessionStats[];
44
+ // Prepared data comparison (if available)
45
+ preparedObservationTokens?: number;
46
+ compressionRatio?: number;
47
+ }
48
+
49
+ interface ObservationOnlyStats {
50
+ questionId: string;
51
+ questionType: string;
52
+ observationTokens: number;
53
+ found: boolean;
54
+ }
55
+
56
+ interface AggregateStats {
57
+ totalQuestions: number;
58
+ totalTokens: number;
59
+ totalMessages: number;
60
+ avgTokensPerQuestion: number;
61
+ minTokens: number;
62
+ maxTokens: number;
63
+ medianTokens: number;
64
+ p90Tokens: number;
65
+ p95Tokens: number;
66
+ distribution: { range: string; count: number }[];
67
+ topQuestions: { questionId: string; tokens: number }[];
68
+ estimatedCost: {
69
+ gpt4o: number;
70
+ gpt4oMini: number;
71
+ claude35Sonnet: number;
72
+ };
73
+ }
74
+
75
+ export class TokensCommand {
76
+ private loader: DatasetLoader;
77
+ private tokenCounter: TokenCounter;
78
+ private preparedDataDir: string;
79
+
80
+ constructor() {
81
+ this.loader = new DatasetLoader();
82
+ this.tokenCounter = new TokenCounter();
83
+ this.preparedDataDir = './prepared-data';
84
+ }
85
+
86
+ async run(options: TokensOptions): Promise<void> {
87
+ // Fast path for observations-only mode
88
+ if (options.observationsOnly) {
89
+ await this.runObservationsOnly(options);
90
+ return;
91
+ }
92
+
93
+ const spinner = ora('Loading dataset...').start();
94
+
95
+ try {
96
+ // Load dataset
97
+ const questions = await this.loader.loadDataset(options.dataset);
98
+ spinner.succeed(`Loaded ${questions.length} questions`);
99
+
100
+ // Filter questions based on options
101
+ let selectedQuestions: LongMemEvalQuestion[];
102
+
103
+ if (options.questionId) {
104
+ const question = questions.find(q => q.question_id === options.questionId);
105
+ if (!question) {
106
+ console.log(chalk.red(`Question ${options.questionId} not found`));
107
+ return;
108
+ }
109
+ selectedQuestions = [question];
110
+ } else {
111
+ const offset = options.offset || 0;
112
+ selectedQuestions = questions.slice(offset);
113
+ if (options.subset) {
114
+ selectedQuestions = selectedQuestions.slice(0, options.subset);
115
+ }
116
+ }
117
+
118
+ console.log(chalk.blue(`\n📊 Token Estimate for ${selectedQuestions.length} question(s)\n`));
119
+
120
+ // Calculate stats for each question
121
+ const questionStats: QuestionStats[] = [];
122
+ const statsSpinner = ora('Calculating token counts...').start();
123
+
124
+ for (let i = 0; i < selectedQuestions.length; i++) {
125
+ const question = selectedQuestions[i];
126
+ statsSpinner.text = `Calculating token counts... ${i + 1}/${selectedQuestions.length}`;
127
+
128
+ const stats = await this.calculateQuestionStats(question, options);
129
+ questionStats.push(stats);
130
+ }
131
+
132
+ statsSpinner.succeed('Token counts calculated');
133
+
134
+ // Display results
135
+ if (selectedQuestions.length === 1) {
136
+ this.displaySingleQuestion(questionStats[0], options.showSessions);
137
+ } else {
138
+ const aggregate = this.calculateAggregateStats(questionStats, options.topN || 10);
139
+ this.displayAggregateStats(aggregate, questionStats, options.showSessions);
140
+ }
141
+ } catch (error) {
142
+ spinner.fail('Error loading dataset');
143
+ throw error;
144
+ }
145
+ }
146
+
147
+ /**
148
+ * Fast path: only count observation tokens from prepared data (skip raw session analysis)
149
+ */
150
+ private async runObservationsOnly(options: TokensOptions): Promise<void> {
151
+ const config = options.config || 'observational-memory';
152
+ const preparedDir = options.preparedDataDir || this.preparedDataDir;
153
+ const configDir = join(preparedDir, options.dataset, config);
154
+
155
+ // Check if config directory exists
156
+ if (!existsSync(configDir)) {
157
+ console.log(chalk.red(`Config directory not found: ${configDir}`));
158
+ console.log(chalk.gray(`Use --config to specify a different config, or prepare data first.`));
159
+ return;
160
+ }
161
+
162
+ const spinner = ora('Scanning prepared data...').start();
163
+
164
+ // Get all question directories
165
+ const { readdirSync } = await import('fs');
166
+ const questionDirs = readdirSync(configDir, { withFileTypes: true })
167
+ .filter(d => d.isDirectory())
168
+ .map(d => d.name);
169
+
170
+ // Filter by questionId if specified
171
+ let targetQuestions = questionDirs;
172
+ if (options.questionId) {
173
+ if (!questionDirs.includes(options.questionId)) {
174
+ spinner.fail(`Question ${options.questionId} not found in ${config}`);
175
+ return;
176
+ }
177
+ targetQuestions = [options.questionId];
178
+ } else {
179
+ // Apply offset and subset
180
+ const offset = options.offset || 0;
181
+ targetQuestions = questionDirs.slice(offset);
182
+ if (options.subset) {
183
+ targetQuestions = targetQuestions.slice(0, options.subset);
184
+ }
185
+ }
186
+
187
+ spinner.text = `Counting observation tokens for ${targetQuestions.length} questions...`;
188
+
189
+ const stats: ObservationOnlyStats[] = [];
190
+ let totalTokens = 0;
191
+ let foundCount = 0;
192
+
193
+ for (const questionId of targetQuestions) {
194
+ const omJsonPath = join(configDir, questionId, 'om.json');
195
+ let observationTokens = 0;
196
+ let found = false;
197
+
198
+ if (existsSync(omJsonPath)) {
199
+ try {
200
+ const omData = JSON.parse(await readFile(omJsonPath, 'utf-8'));
201
+
202
+ // Sum activeObservations across ALL records
203
+ if (Array.isArray(omData.observationalMemory) && omData.observationalMemory[0]) {
204
+ const [, records] = omData.observationalMemory[0];
205
+ if (Array.isArray(records)) {
206
+ for (const record of records) {
207
+ if (record?.activeObservations) {
208
+ observationTokens += this.tokenCounter.countString(record.activeObservations);
209
+ }
210
+ }
211
+ }
212
+ } else if (omData.record?.activeObservations) {
213
+ observationTokens = this.tokenCounter.countString(omData.record.activeObservations);
214
+ }
215
+
216
+ if (observationTokens > 0) {
217
+ found = true;
218
+ foundCount++;
219
+ totalTokens += observationTokens;
220
+ }
221
+ } catch {
222
+ // Ignore parse errors
223
+ }
224
+ }
225
+
226
+ // Try to get question type from meta.json
227
+ let questionType = 'unknown';
228
+ const metaPath = join(configDir, questionId, 'meta.json');
229
+ if (existsSync(metaPath)) {
230
+ try {
231
+ const meta = JSON.parse(await readFile(metaPath, 'utf-8'));
232
+ questionType = meta.questionType || 'unknown';
233
+ } catch {
234
+ // Ignore
235
+ }
236
+ }
237
+
238
+ stats.push({ questionId, questionType, observationTokens, found });
239
+ }
240
+
241
+ spinner.succeed(`Counted ${foundCount}/${targetQuestions.length} questions with observations`);
242
+
243
+ // Display results
244
+ console.log(chalk.blue(`\n📊 Observation Tokens (${config})\n`));
245
+
246
+ if (targetQuestions.length === 1) {
247
+ // Single question view
248
+ const s = stats[0];
249
+ console.log(chalk.bold(`Question: ${s.questionId}`));
250
+ console.log(chalk.gray(`Type: ${s.questionType}`));
251
+ console.log();
252
+ if (s.found) {
253
+ console.log(` Observation tokens: ${this.formatTokens(s.observationTokens)}`);
254
+ } else {
255
+ console.log(chalk.yellow(` No observations found`));
256
+ }
257
+ } else {
258
+ // Aggregate view
259
+ const tokenCounts = stats
260
+ .filter(s => s.found)
261
+ .map(s => s.observationTokens)
262
+ .sort((a, b) => a - b);
263
+
264
+ const percentile = (arr: number[], p: number) => {
265
+ if (arr.length === 0) return 0;
266
+ const idx = Math.ceil((p / 100) * arr.length) - 1;
267
+ return arr[Math.max(0, idx)];
268
+ };
269
+
270
+ console.log(chalk.bold('Summary:'));
271
+ console.log(` Questions with data: ${foundCount}/${targetQuestions.length}`);
272
+ console.log(` Total tokens: ${this.formatTokens(totalTokens)}`);
273
+ console.log(
274
+ ` Avg tokens/question: ${this.formatTokens(foundCount > 0 ? Math.round(totalTokens / foundCount) : 0)}`,
275
+ );
276
+ console.log();
277
+
278
+ if (tokenCounts.length > 0) {
279
+ console.log(chalk.bold('Distribution:'));
280
+ console.log(` Min: ${this.formatTokens(tokenCounts[0])}`);
281
+ console.log(` Median: ${this.formatTokens(percentile(tokenCounts, 50))}`);
282
+ console.log(` P90: ${this.formatTokens(percentile(tokenCounts, 90))}`);
283
+ console.log(` P95: ${this.formatTokens(percentile(tokenCounts, 95))}`);
284
+ console.log(` Max: ${this.formatTokens(tokenCounts[tokenCounts.length - 1])}`);
285
+ console.log();
286
+
287
+ // Top N largest
288
+ const topN = options.topN || 10;
289
+ const sorted = [...stats].filter(s => s.found).sort((a, b) => b.observationTokens - a.observationTokens);
290
+ console.log(chalk.bold(`Top ${Math.min(topN, sorted.length)} Largest:`));
291
+ for (let i = 0; i < Math.min(topN, sorted.length); i++) {
292
+ const s = sorted[i];
293
+ console.log(
294
+ ` ${(i + 1).toString().padStart(2)}. ${s.questionId} ${this.formatTokens(s.observationTokens)}`,
295
+ );
296
+ }
297
+ console.log();
298
+
299
+ // Cost estimates
300
+ const gpt4oInputPrice = 2.5;
301
+ const gpt4oMiniInputPrice = 0.15;
302
+ console.log(chalk.bold('Estimated Input Costs:'));
303
+ console.log(` GPT-4o: $${((totalTokens / 1_000_000) * gpt4oInputPrice).toFixed(2)}`);
304
+ console.log(` GPT-4o-mini: $${((totalTokens / 1_000_000) * gpt4oMiniInputPrice).toFixed(2)}`);
305
+ }
306
+ }
307
+
308
+ console.log();
309
+ }
310
+
311
+ private async calculateQuestionStats(question: LongMemEvalQuestion, options: TokensOptions): Promise<QuestionStats> {
312
+ const sessions: SessionStats[] = [];
313
+ const answerSessionIdSet = new Set(question.answer_session_ids || []);
314
+
315
+ // haystack_sessions is Turn[][] and haystack_session_ids is string[]
316
+ // They are parallel arrays
317
+ for (let i = 0; i < question.haystack_sessions.length; i++) {
318
+ const sessionId = question.haystack_session_ids[i];
319
+ const messages = question.haystack_sessions[i];
320
+ const isAnswerSession = answerSessionIdSet.has(sessionId);
321
+
322
+ // Count tokens in messages
323
+ let tokens = 0;
324
+ let userTokens = 0;
325
+ let assistantTokens = 0;
326
+ let messageCount = 0;
327
+
328
+ for (const msg of messages) {
329
+ // Count tokens directly from the string content
330
+ // Add role overhead (~4 tokens) to match TokenCounter.countMessage behavior
331
+ const contentTokens = this.tokenCounter.countString(msg.content);
332
+ const roleTokens = this.tokenCounter.countString(msg.role);
333
+ const msgTokens = contentTokens + roleTokens + 4; // ~4 tokens overhead per message
334
+
335
+ tokens += msgTokens;
336
+ messageCount++;
337
+
338
+ if (msg.role === 'user') {
339
+ userTokens += msgTokens;
340
+ } else {
341
+ assistantTokens += msgTokens;
342
+ }
343
+ }
344
+
345
+ sessions.push({
346
+ sessionId,
347
+ isAnswerSession,
348
+ tokens,
349
+ messageCount,
350
+ userTokens,
351
+ assistantTokens,
352
+ });
353
+ }
354
+
355
+ // Calculate aggregates
356
+ const totalTokens = sessions.reduce((sum, s) => sum + s.tokens, 0);
357
+ const answerSessionTokens = sessions.filter(s => s.isAnswerSession).reduce((sum, s) => sum + s.tokens, 0);
358
+ const messageCount = sessions.reduce((sum, s) => sum + s.messageCount, 0);
359
+
360
+ const sortedByTokens = [...sessions].sort((a, b) => b.tokens - a.tokens);
361
+ const largestSession = sortedByTokens[0] || { id: '', tokens: 0 };
362
+ const smallestSession = sortedByTokens[sortedByTokens.length - 1] || { id: '', tokens: 0 };
363
+
364
+ // Check for prepared data
365
+ let preparedObservationTokens: number | undefined;
366
+ let compressionRatio: number | undefined;
367
+
368
+ const preparedDir = options.preparedDataDir || this.preparedDataDir;
369
+ // Check all observational-memory config directories
370
+ const omConfigs = ['observational-memory', 'observational-memory-shortcut', 'observational-memory-shortcut-glm'];
371
+ let omJsonPath = '';
372
+ for (const config of omConfigs) {
373
+ const candidatePath = join(preparedDir, options.dataset, config, question.question_id, 'om.json');
374
+ if (existsSync(candidatePath)) {
375
+ omJsonPath = candidatePath;
376
+ break;
377
+ }
378
+ }
379
+
380
+ if (omJsonPath && existsSync(omJsonPath)) {
381
+ try {
382
+ const omData = JSON.parse(await readFile(omJsonPath, 'utf-8'));
383
+ // The structure is: observationalMemory[0] = [resourceKey, [records...]]
384
+ // Sum activeObservations across ALL records (each reflection creates a new record)
385
+ let totalObservationTokens = 0;
386
+
387
+ if (Array.isArray(omData.observationalMemory) && omData.observationalMemory[0]) {
388
+ const [, records] = omData.observationalMemory[0];
389
+ if (Array.isArray(records)) {
390
+ for (const record of records) {
391
+ if (record?.activeObservations) {
392
+ totalObservationTokens += this.tokenCounter.countString(record.activeObservations);
393
+ }
394
+ }
395
+ }
396
+ } else if (omData.record?.activeObservations) {
397
+ // Fallback for old structure
398
+ totalObservationTokens = this.tokenCounter.countString(omData.record.activeObservations);
399
+ }
400
+
401
+ if (totalObservationTokens > 0) {
402
+ preparedObservationTokens = totalObservationTokens;
403
+ if (totalTokens > 0) {
404
+ compressionRatio = totalTokens / preparedObservationTokens;
405
+ }
406
+ }
407
+ } catch {
408
+ // Ignore errors reading prepared data
409
+ }
410
+ }
411
+
412
+ return {
413
+ questionId: question.question_id,
414
+ questionType: question.question_type,
415
+ totalTokens,
416
+ answerSessionTokens,
417
+ messageCount,
418
+ sessionCount: sessions.length,
419
+ answerSessionCount: sessions.filter(s => s.isAnswerSession).length,
420
+ largestSession: { id: largestSession.sessionId, tokens: largestSession.tokens },
421
+ smallestSession: { id: smallestSession.sessionId, tokens: smallestSession.tokens },
422
+ avgTokensPerMessage: messageCount > 0 ? Math.round(totalTokens / messageCount) : 0,
423
+ sessions,
424
+ preparedObservationTokens,
425
+ compressionRatio,
426
+ };
427
+ }
428
+
429
+ private calculateAggregateStats(questionStats: QuestionStats[], topN: number): AggregateStats {
430
+ const tokenCounts = questionStats.map(q => q.totalTokens).sort((a, b) => a - b);
431
+ const totalTokens = tokenCounts.reduce((sum, t) => sum + t, 0);
432
+ const totalMessages = questionStats.reduce((sum, q) => sum + q.messageCount, 0);
433
+
434
+ // Percentiles
435
+ const percentile = (arr: number[], p: number) => {
436
+ const idx = Math.ceil((p / 100) * arr.length) - 1;
437
+ return arr[Math.max(0, idx)];
438
+ };
439
+
440
+ // Distribution buckets
441
+ const buckets = [
442
+ { min: 0, max: 10000, label: '0-10k' },
443
+ { min: 10000, max: 25000, label: '10k-25k' },
444
+ { min: 25000, max: 50000, label: '25k-50k' },
445
+ { min: 50000, max: 100000, label: '50k-100k' },
446
+ { min: 100000, max: 250000, label: '100k-250k' },
447
+ { min: 250000, max: Infinity, label: '250k+' },
448
+ ];
449
+
450
+ const distribution = buckets.map(bucket => ({
451
+ range: bucket.label,
452
+ count: questionStats.filter(q => q.totalTokens >= bucket.min && q.totalTokens < bucket.max).length,
453
+ }));
454
+
455
+ // Top N largest questions
456
+ const topQuestions = [...questionStats]
457
+ .sort((a, b) => b.totalTokens - a.totalTokens)
458
+ .slice(0, topN)
459
+ .map(q => ({ questionId: q.questionId, tokens: q.totalTokens }));
460
+
461
+ // Cost estimates (per 1M tokens)
462
+ const gpt4oInputPrice = 2.5; // $2.50 per 1M input tokens
463
+ const gpt4oMiniInputPrice = 0.15; // $0.15 per 1M input tokens
464
+ const claude35SonnetInputPrice = 3.0; // $3.00 per 1M input tokens
465
+
466
+ return {
467
+ totalQuestions: questionStats.length,
468
+ totalTokens,
469
+ totalMessages,
470
+ avgTokensPerQuestion: Math.round(totalTokens / questionStats.length),
471
+ minTokens: tokenCounts[0] || 0,
472
+ maxTokens: tokenCounts[tokenCounts.length - 1] || 0,
473
+ medianTokens: percentile(tokenCounts, 50),
474
+ p90Tokens: percentile(tokenCounts, 90),
475
+ p95Tokens: percentile(tokenCounts, 95),
476
+ distribution,
477
+ topQuestions,
478
+ estimatedCost: {
479
+ gpt4o: (totalTokens / 1_000_000) * gpt4oInputPrice,
480
+ gpt4oMini: (totalTokens / 1_000_000) * gpt4oMiniInputPrice,
481
+ claude35Sonnet: (totalTokens / 1_000_000) * claude35SonnetInputPrice,
482
+ },
483
+ };
484
+ }
485
+
486
+ private displaySingleQuestion(stats: QuestionStats, showSessions?: boolean): void {
487
+ console.log(chalk.bold(`Question: ${stats.questionId}`));
488
+ console.log(chalk.gray(`Type: ${stats.questionType}`));
489
+ console.log();
490
+
491
+ if (showSessions) {
492
+ console.log(chalk.bold(`Sessions (${stats.sessionCount} total):\n`));
493
+
494
+ // Sort sessions by tokens descending
495
+ const sortedSessions = [...stats.sessions].sort((a, b) => b.tokens - a.tokens);
496
+
497
+ for (const session of sortedSessions) {
498
+ const answerTag = session.isAnswerSession ? chalk.green(' (answer)') : '';
499
+ const tokens = this.formatTokens(session.tokens);
500
+ const breakdown = chalk.gray(
501
+ `(${session.messageCount} msgs, U:${this.formatTokens(session.userTokens)} A:${this.formatTokens(session.assistantTokens)})`,
502
+ );
503
+
504
+ console.log(` ${session.sessionId}${answerTag}`);
505
+ console.log(` ${tokens} ${breakdown}`);
506
+ }
507
+ console.log();
508
+ }
509
+
510
+ console.log(chalk.bold('Summary:'));
511
+ console.log(` Total tokens: ${this.formatTokens(stats.totalTokens)}`);
512
+ console.log(
513
+ ` Answer session tokens: ${this.formatTokens(stats.answerSessionTokens)} ${chalk.gray(`(${stats.answerSessionCount} sessions)`)}`,
514
+ );
515
+ console.log(` Total messages: ${stats.messageCount}`);
516
+ console.log(` Avg tokens/message: ${stats.avgTokensPerMessage}`);
517
+ console.log(
518
+ ` Largest session: ${stats.largestSession.id} (${this.formatTokens(stats.largestSession.tokens)})`,
519
+ );
520
+ console.log(
521
+ ` Smallest session: ${stats.smallestSession.id} (${this.formatTokens(stats.smallestSession.tokens)})`,
522
+ );
523
+
524
+ if (stats.preparedObservationTokens !== undefined) {
525
+ console.log();
526
+ console.log(chalk.bold('Prepared Data (OM):'));
527
+ console.log(` Observation tokens: ${this.formatTokens(stats.preparedObservationTokens)}`);
528
+ if (stats.compressionRatio !== undefined) {
529
+ console.log(` Compression ratio: ${stats.compressionRatio.toFixed(1)}x`);
530
+ }
531
+ }
532
+
533
+ console.log();
534
+ }
535
+
536
+ private displayAggregateStats(
537
+ aggregate: AggregateStats,
538
+ questionStats: QuestionStats[],
539
+ showSessions?: boolean,
540
+ ): void {
541
+ // Summary
542
+ console.log(chalk.bold('Summary:'));
543
+ console.log(` Questions: ${aggregate.totalQuestions}`);
544
+ console.log(` Total tokens: ${this.formatTokens(aggregate.totalTokens)}`);
545
+ console.log(` Total messages: ${aggregate.totalMessages.toLocaleString()}`);
546
+ console.log(` Avg tokens/question: ${this.formatTokens(aggregate.avgTokensPerQuestion)}`);
547
+ console.log();
548
+
549
+ // Distribution
550
+ console.log(chalk.bold('Token Distribution:'));
551
+ console.log(` Min: ${this.formatTokens(aggregate.minTokens)}`);
552
+ console.log(` Median: ${this.formatTokens(aggregate.medianTokens)}`);
553
+ console.log(` P90: ${this.formatTokens(aggregate.p90Tokens)}`);
554
+ console.log(` P95: ${this.formatTokens(aggregate.p95Tokens)}`);
555
+ console.log(` Max: ${this.formatTokens(aggregate.maxTokens)}`);
556
+ console.log();
557
+
558
+ // Histogram
559
+ console.log(chalk.bold('Distribution by Range:'));
560
+ const maxCount = Math.max(...aggregate.distribution.map(d => d.count));
561
+ const barWidth = 30;
562
+
563
+ for (const bucket of aggregate.distribution) {
564
+ if (bucket.count === 0) continue;
565
+ const barLength = Math.round((bucket.count / maxCount) * barWidth);
566
+ const bar = '█'.repeat(barLength);
567
+ const pct = ((bucket.count / aggregate.totalQuestions) * 100).toFixed(0);
568
+ console.log(` ${bucket.range.padEnd(10)} ${bar} ${bucket.count} (${pct}%)`);
569
+ }
570
+ console.log();
571
+
572
+ // Top N largest
573
+ console.log(chalk.bold(`Top ${aggregate.topQuestions.length} Largest Questions:`));
574
+ for (let i = 0; i < aggregate.topQuestions.length; i++) {
575
+ const q = aggregate.topQuestions[i];
576
+ console.log(` ${(i + 1).toString().padStart(2)}. ${q.questionId} ${this.formatTokens(q.tokens)}`);
577
+ }
578
+ console.log();
579
+
580
+ // Cost estimates
581
+ console.log(chalk.bold('Estimated Input Costs:'));
582
+ console.log(` GPT-4o: $${aggregate.estimatedCost.gpt4o.toFixed(2)}`);
583
+ console.log(` GPT-4o-mini: $${aggregate.estimatedCost.gpt4oMini.toFixed(2)}`);
584
+ console.log(` Claude 3.5 Sonnet: $${aggregate.estimatedCost.claude35Sonnet.toFixed(2)}`);
585
+ console.log(chalk.gray(' (Input tokens only, excludes output/observation costs)'));
586
+ console.log();
587
+
588
+ // Compression stats if any have prepared data
589
+ const withPrepared = questionStats.filter(q => q.preparedObservationTokens !== undefined);
590
+ if (withPrepared.length > 0) {
591
+ const totalPreparedTokens = withPrepared.reduce((sum, q) => sum + (q.preparedObservationTokens || 0), 0);
592
+ const totalOriginalTokens = withPrepared.reduce((sum, q) => sum + q.totalTokens, 0);
593
+ const avgCompression = totalOriginalTokens / totalPreparedTokens;
594
+
595
+ console.log(chalk.bold('Prepared Data (OM):'));
596
+ console.log(` Questions with OM: ${withPrepared.length}`);
597
+ console.log(` Total OM tokens: ${this.formatTokens(totalPreparedTokens)}`);
598
+ console.log(` Avg compression: ${avgCompression.toFixed(1)}x`);
599
+ console.log();
600
+ }
601
+
602
+ // Show per-question details if requested
603
+ if (showSessions) {
604
+ console.log(chalk.bold('Per-Question Details:\n'));
605
+ for (const stats of questionStats) {
606
+ const compression = stats.compressionRatio
607
+ ? chalk.green(` (${stats.compressionRatio.toFixed(1)}x compression)`)
608
+ : '';
609
+ console.log(
610
+ ` ${stats.questionId} - ${this.formatTokens(stats.totalTokens)} (${stats.sessionCount} sessions, ${stats.messageCount} msgs)${compression}`,
611
+ );
612
+ }
613
+ console.log();
614
+ }
615
+ }
616
+
617
+ private formatTokens(tokens: number): string {
618
+ if (tokens >= 1_000_000) {
619
+ return `${(tokens / 1_000_000).toFixed(2)}M`;
620
+ } else if (tokens >= 1_000) {
621
+ return `${(tokens / 1_000).toFixed(1)}k`;
622
+ }
623
+ return tokens.toString();
624
+ }
625
+ }