@mastra/longmemeval 0.0.0-error-handler-fix-20251020202607 → 0.0.0-execa-dynamic-import-20260304221256

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/CHANGELOG.md +847 -6
  2. package/LICENSE.md +15 -0
  3. package/docs/INVESTIGATION_WORKFLOW.md +314 -0
  4. package/package.json +34 -26
  5. package/scripts/download.ts +22 -1
  6. package/scripts/generate-wm-templates.ts +2 -1
  7. package/src/cli.ts +967 -155
  8. package/src/commands/__tests__/prepare.test.ts +11 -6
  9. package/src/commands/__tests__/run.test.ts +10 -8
  10. package/src/commands/clean.ts +147 -0
  11. package/src/commands/deterministic-ids.ts +126 -0
  12. package/src/commands/find-prohibited.ts +299 -0
  13. package/src/commands/investigate.ts +2437 -0
  14. package/src/commands/list-partial.ts +137 -0
  15. package/src/commands/obscure-thread-ids.ts +237 -0
  16. package/src/commands/partial-results.ts +198 -0
  17. package/src/commands/precompute-embeddings.ts +302 -0
  18. package/src/commands/prepare.ts +732 -105
  19. package/src/commands/reconcile.ts +383 -0
  20. package/src/commands/run.ts +1240 -121
  21. package/src/commands/sessions.ts +126 -0
  22. package/src/commands/sync.ts +152 -0
  23. package/src/commands/test-cleaned.ts +53 -0
  24. package/src/commands/test-incremental.ts +120 -0
  25. package/src/commands/test-messages.ts +95 -0
  26. package/src/commands/test-prohibited.ts +244 -0
  27. package/src/commands/test-thread.ts +136 -0
  28. package/src/commands/tokens.ts +625 -0
  29. package/src/config.ts +1253 -72
  30. package/src/data/types.ts +48 -6
  31. package/src/embeddings/cached-openai-embedding-model.ts +0 -1
  32. package/src/evaluation/longmemeval-metric.ts +186 -103
  33. package/src/processors/content-sanitizer.ts +153 -0
  34. package/src/processors/date-injector.ts +111 -0
  35. package/src/processors/index.ts +3 -0
  36. package/src/processors/observation-semantic-filter.ts +1142 -0
  37. package/src/retry-model.ts +5 -3
  38. package/src/storage/__tests__/benchmark-store.test.ts +18 -35
  39. package/src/storage/benchmark-store.ts +140 -153
  40. package/src/storage/benchmark-vector.ts +1 -1
  41. package/src/storage/index.ts +1 -0
  42. package/src/storage/persistable-inmemory.ts +133 -0
  43. package/src/test-utils/mock-model.ts +35 -0
@@ -1,28 +1,255 @@
1
1
  import { Agent } from '@mastra/core/agent';
2
+ import { RequestContext } from '@mastra/core/request-context';
2
3
  import { Memory } from '@mastra/memory';
3
- import { openai } from '@ai-sdk/openai';
4
+ import { ObservationalMemory, OBSERVATIONAL_MEMORY_DEFAULTS } from '@mastra/memory/processors';
4
5
  import { cachedOpenAI } from '../embeddings/cached-openai-provider';
5
6
  import chalk from 'chalk';
6
7
  import ora, { Ora } from 'ora';
7
8
  import { join } from 'path';
8
- import { readdir, readFile, mkdir, writeFile } from 'fs/promises';
9
- import { existsSync } from 'fs';
9
+ import { readdir, readFile, mkdir, writeFile, stat, appendFile } from 'fs/promises';
10
+ import { appendFileSync, existsSync, writeFileSync, mkdirSync } from 'fs';
11
+
12
+ /**
13
+ * Parse duration string (e.g., "1h", "30m", "2d") to milliseconds
14
+ */
15
+ function parseDuration(duration: string): number {
16
+ const match = duration.match(/^(\d+)(m|h|d)$/);
17
+ if (!match) {
18
+ throw new Error(`Invalid duration format: ${duration}. Use format like "1h", "30m", "2d"`);
19
+ }
20
+ const value = parseInt(match[1], 10);
21
+ const unit = match[2];
22
+ switch (unit) {
23
+ case 'm':
24
+ return value * 60 * 1000;
25
+ case 'h':
26
+ return value * 60 * 60 * 1000;
27
+ case 'd':
28
+ return value * 24 * 60 * 60 * 1000;
29
+ default:
30
+ throw new Error(`Unknown duration unit: ${unit}`);
31
+ }
32
+ }
33
+
34
+ interface FailuresFile {
35
+ questionIds: string[];
36
+ runId: string;
37
+ config: MemoryConfigType;
38
+ dataset: string;
39
+ timestamp: string;
40
+ totalFailed: number;
41
+ totalQuestions: number;
42
+ }
10
43
 
11
- import { BenchmarkStore, BenchmarkVectorStore } from '../storage';
44
+ import { BenchmarkStore, BenchmarkVectorStore, PersistableInMemoryMemory } from '../storage';
12
45
  import { LongMemEvalMetric } from '../evaluation/longmemeval-metric';
13
46
  import type { EvaluationResult, BenchmarkMetrics, QuestionType, MemoryConfigType, DatasetType } from '../data/types';
14
- import { getMemoryOptions } from '../config';
15
- import { makeRetryModel } from '../retry-model';
47
+ import {
48
+ getMemoryConfig,
49
+ getMemoryOptions,
50
+ applyStratifiedSampling,
51
+ applyCombSampling,
52
+ type MemoryConfigDefinition,
53
+ } from '../config';
54
+ import { DatasetLoader } from '../data/loader';
55
+ import { reconcileQuestion } from './reconcile';
56
+ import { ObservationSemanticFilter, DateInjector } from '../processors';
57
+ import { fastembed } from '@mastra/fastembed';
58
+
59
+ // Rate limit configuration
60
+ const RATE_LIMIT_TOKEN_THRESHOLD = 50000; // Pause if remaining tokens below this
61
+
62
+ // Shared rate limiter state across all workers
63
+ const rateLimiter = {
64
+ remainingTokens: 999999,
65
+ remainingRequests: 999999,
66
+ isWaiting: false,
67
+ waitPromise: null as Promise<void> | null,
68
+ };
69
+
70
+ /**
71
+ * Update rate limiter state from response headers
72
+ */
73
+ function updateRateLimiterFromResponse(response: any): void {
74
+ const headers = response?.response?.headers;
75
+ if (!headers) return;
76
+
77
+ rateLimiter.remainingTokens = parseInt(headers['x-ratelimit-remaining-tokens-minute'] || '999999', 10);
78
+ rateLimiter.remainingRequests = parseInt(headers['x-ratelimit-remaining-requests-minute'] || '999999', 10);
79
+ }
80
+
81
+ /**
82
+ * Wait for rate limit to reset (shared across all workers)
83
+ */
84
+ async function waitForRateLimit(waitSeconds: number, reason: string): Promise<void> {
85
+ // If already waiting, just join the existing wait
86
+ if (rateLimiter.isWaiting && rateLimiter.waitPromise) {
87
+ return rateLimiter.waitPromise;
88
+ }
89
+
90
+ rateLimiter.isWaiting = true;
91
+ console.log(
92
+ chalk.yellow(`
93
+ ⏳ ${reason}. Waiting ${waitSeconds}s...`),
94
+ );
95
+
96
+ rateLimiter.waitPromise = (async () => {
97
+ await new Promise(resolve => setTimeout(resolve, waitSeconds * 1000));
98
+ rateLimiter.isWaiting = false;
99
+ rateLimiter.remainingTokens = 999999; // Reset after wait
100
+ rateLimiter.remainingRequests = 999999;
101
+ console.log(chalk.green(`✓ Resuming after rate limit pause`));
102
+ })();
103
+
104
+ return rateLimiter.waitPromise;
105
+ }
106
+
107
+ /**
108
+ * Check rate limit before making a request
109
+ */
110
+ async function checkRateLimitBeforeRequest(): Promise<void> {
111
+ // If already waiting, join the wait
112
+ if (rateLimiter.isWaiting && rateLimiter.waitPromise) {
113
+ await rateLimiter.waitPromise;
114
+ return;
115
+ }
116
+
117
+ // Check if we're approaching limits
118
+ if (rateLimiter.remainingTokens < RATE_LIMIT_TOKEN_THRESHOLD || rateLimiter.remainingRequests < 5) {
119
+ await waitForRateLimit(
120
+ 60,
121
+ `Rate limit approaching (${rateLimiter.remainingTokens} tokens, ${rateLimiter.remainingRequests} requests remaining)`,
122
+ );
123
+ }
124
+ }
125
+
126
+ /**
127
+ * Wrapper to handle 429 errors with retry
128
+ */
129
+ async function withRateLimitRetry<T>(
130
+ fn: () => Promise<T>,
131
+ spinner?: Ora | { updateStatus: (status: string) => void },
132
+ maxRetries = 3,
133
+ ): Promise<T> {
134
+ for (let attempt = 0; attempt <= maxRetries; attempt++) {
135
+ // Check rate limit before request
136
+ await checkRateLimitBeforeRequest();
137
+
138
+ try {
139
+ const result = await fn();
140
+ // Update rate limiter from successful response
141
+ updateRateLimiterFromResponse(result);
142
+ return result;
143
+ } catch (error: any) {
144
+ // Check if it's a 429 rate limit error (may be nested in MastraError)
145
+ const is429 =
146
+ error?.statusCode === 429 ||
147
+ error?.cause?.statusCode === 429 ||
148
+ error?.message?.includes('Too Many Requests') ||
149
+ error?.message?.includes('Rate limit') ||
150
+ error?.message?.includes('rate_limit');
151
+
152
+ // Check if it's a retryable connection error (ECONNRESET, ETIMEDOUT, etc.)
153
+ const isConnectionError =
154
+ error?.message?.includes('ECONNRESET') ||
155
+ error?.message?.includes('ETIMEDOUT') ||
156
+ error?.message?.includes('ENOTFOUND') ||
157
+ error?.message?.includes('ECONNREFUSED') ||
158
+ error?.message?.includes('Cannot connect to API') ||
159
+ error?.isRetryable === true ||
160
+ error?.cause?.code === 'ECONNRESET' ||
161
+ error?.cause?.code === 'ETIMEDOUT';
162
+
163
+ if (isConnectionError && !is429) {
164
+ // Connection error - wait a shorter time and retry
165
+ const waitTime = 10 + attempt * 5; // 10s, 15s, 20s...
166
+
167
+ const updateStatus = (status: string) => {
168
+ if (spinner && 'updateStatus' in spinner) {
169
+ spinner.updateStatus(status);
170
+ } else if (spinner && 'text' in spinner) {
171
+ spinner.text = status;
172
+ }
173
+ };
174
+ updateStatus(`Connection error - waiting ${waitTime}s (attempt ${attempt + 1}/${maxRetries})...`);
175
+ console.error(
176
+ `\n🔌 Connection error (${error?.cause?.code || 'unknown'}) - waiting ${waitTime}s before retry ${attempt + 1}/${maxRetries}`,
177
+ );
178
+
179
+ await waitForRateLimit(waitTime, `Connection error`);
180
+
181
+ if (attempt < maxRetries) {
182
+ continue; // Retry
183
+ }
184
+ }
185
+
186
+ if (is429) {
187
+ // Try to extract retry-after from nested error (check various header formats)
188
+ const headers = error?.responseHeaders || error?.cause?.responseHeaders || {};
189
+ let retryAfter = 60; // Default to 60 seconds
190
+
191
+ if (headers['retry-after']) {
192
+ retryAfter = parseInt(headers['retry-after'], 10);
193
+ } else if (headers['x-ratelimit-reset-tokens']) {
194
+ // Parse OpenAI format like "1m2.251s" or "4ms"
195
+ const resetStr = headers['x-ratelimit-reset-tokens'];
196
+ const minutes = resetStr.match(/(\d+)m(?!\s*s)/)?.[1];
197
+ const seconds = resetStr.match(/(\d+\.?\d*)s/)?.[1];
198
+ const ms = resetStr.match(/(\d+)ms/)?.[1];
199
+ retryAfter =
200
+ parseInt(minutes || '0', 10) * 60 +
201
+ Math.ceil(parseFloat(seconds || '0')) +
202
+ Math.ceil(parseInt(ms || '0', 10) / 1000);
203
+ retryAfter = Math.max(retryAfter, 5); // At least 5 seconds
204
+ }
205
+
206
+ // Add a small buffer to avoid hitting the limit again immediately
207
+ const waitTime = retryAfter + 5;
208
+
209
+ const updateStatus = (status: string) => {
210
+ if (spinner && 'updateStatus' in spinner) {
211
+ spinner.updateStatus(status);
212
+ } else if (spinner && 'text' in spinner) {
213
+ spinner.text = status;
214
+ }
215
+ };
216
+ updateStatus(`Rate limited (429) - waiting ${waitTime}s (attempt ${attempt + 1}/${maxRetries})...`);
217
+ console.error(`\n⏳ Rate limited by OpenAI - waiting ${waitTime}s before retry ${attempt + 1}/${maxRetries}`);
218
+
219
+ await waitForRateLimit(waitTime, `Rate limited (429 error)`);
220
+
221
+ if (attempt < maxRetries) {
222
+ continue; // Retry
223
+ }
224
+ }
225
+ throw error; // Re-throw if not rate limit or max retries exceeded
226
+ }
227
+ }
228
+ throw new Error('Max retries exceeded');
229
+ }
16
230
 
17
231
  export interface RunOptions {
18
232
  dataset: DatasetType;
19
233
  memoryConfig: MemoryConfigType;
20
- model: string;
21
234
  preparedDataDir?: string;
22
235
  outputDir?: string;
23
236
  subset?: number;
237
+ perTypeCount?: number;
238
+ offset?: number;
24
239
  concurrency?: number;
25
240
  questionId?: string;
241
+ questionType?: string;
242
+ // Comb sampling options
243
+ combSampleSize?: number;
244
+ combOffset?: number;
245
+ combStartOffset?: number;
246
+ // Skip improved/fixed question evaluation
247
+ skipFixed?: boolean;
248
+ // Re-run from failures
249
+ fromFailures?: string | boolean;
250
+ olderThan?: string;
251
+ // Resume a partial run
252
+ resume?: string | boolean;
26
253
  }
27
254
 
28
255
  interface PreparedQuestionMeta {
@@ -32,56 +259,153 @@ interface PreparedQuestionMeta {
32
259
  threadIds: string[];
33
260
  memoryConfig: string;
34
261
  question: string;
262
+ improvedQuestion?: string; // Clarified version for vague/ambiguous questions
263
+ improvedAnswer?: string; // Updated answer for the clarified question (if different)
264
+ improvementNote?: string; // Notes about why this question failed (for tracking investigated failures)
265
+ requiresRetry?: boolean; // Eval agent sometimes fails due to poor reasoning, retry once on failure
35
266
  answer: string;
36
267
  evidenceSessionIds?: string[];
37
268
  questionDate?: string;
38
269
  }
39
270
 
40
- const retry4o = makeRetryModel(openai('gpt-4o'));
271
+ /**
272
+ * Normalize model ID to provider/model format for ModelRouterLanguageModel
273
+ */
274
+ function normalizeModelId(modelId: string): `${string}/${string}` {
275
+ return (modelId.includes('/') ? modelId : `openai/${modelId}`) as `${string}/${string}`;
276
+ }
41
277
 
42
278
  export class RunCommand {
43
279
  private preparedDataDir: string;
44
280
  private outputDir: string;
281
+ private loader: DatasetLoader;
45
282
 
46
283
  constructor() {
47
284
  this.preparedDataDir = './prepared-data';
48
285
  this.outputDir = './results';
286
+ this.loader = new DatasetLoader();
49
287
  }
50
288
 
51
289
  async run(options: RunOptions): Promise<BenchmarkMetrics> {
52
- const runId = `run_${Date.now()}`;
53
- const runDir = join(options.outputDir || this.outputDir, options.memoryConfig, runId);
290
+ // Use existing run ID for resume, or generate new one
291
+ let runId = options.resume;
292
+
293
+ // If --resume is passed without a value (true) or empty string, find the most recent run
294
+ if (options.resume === true || options.resume === '') {
295
+ const configDir = join(options.outputDir || this.outputDir, options.memoryConfig);
296
+ if (existsSync(configDir)) {
297
+ const runs = (await readdir(configDir))
298
+ .filter(d => d.startsWith('run_'))
299
+ .sort()
300
+ .reverse(); // Most recent first (timestamps sort correctly)
301
+ if (runs.length > 0) {
302
+ runId = runs[0];
303
+ console.log(chalk.yellow(`\n🔍 Auto-detected most recent run: ${runId}\n`));
304
+ }
305
+ }
306
+ }
307
+
308
+ // Fall back to new run ID if no resume target found
309
+ if (!runId || runId === true) {
310
+ runId = `run_${Date.now()}`;
311
+ }
312
+
313
+ const runDir = join(options.outputDir || this.outputDir, options.memoryConfig, runId as string);
54
314
  await mkdir(runDir, { recursive: true });
55
315
 
56
- console.log(chalk.blue(`\n🚀 Starting LongMemEval benchmark run: ${runId}\n`));
316
+ // Load existing results if resuming
317
+ let existingResults: EvaluationResult[] = [];
318
+ const resultsPath = join(runDir, 'results.jsonl');
319
+ if (options.resume && existsSync(resultsPath)) {
320
+ const content = await readFile(resultsPath, 'utf-8');
321
+ existingResults = content
322
+ .trim()
323
+ .split('\n')
324
+ .filter(line => line.trim())
325
+ .map(line => JSON.parse(line) as EvaluationResult);
326
+ console.log(chalk.yellow(`\n♻️ Resuming run ${runId} with ${existingResults.length} existing results\n`));
327
+ }
328
+ const completedQuestionIds = new Set(existingResults.map(r => r.question_id));
329
+
330
+ console.log(
331
+ chalk.blue(`
332
+ 🚀 ${options.resume ? 'Resuming' : 'Starting'} LongMemEval benchmark run: ${runId}
333
+ `),
334
+ );
335
+ const configDef = getMemoryConfig(options.memoryConfig);
336
+
57
337
  console.log(chalk.gray(`Dataset: ${options.dataset}`));
58
- console.log(chalk.gray(`Model: ${options.model}`));
338
+ console.log(chalk.gray(`Model: ${configDef.agentModel}`));
59
339
  console.log(chalk.gray(`Memory Config: ${options.memoryConfig}`));
60
340
  if (options.subset) {
61
341
  console.log(chalk.gray(`Subset: ${options.subset} questions`));
62
342
  }
63
343
  console.log();
64
344
 
65
- const preparedDir = join(options.preparedDataDir || this.preparedDataDir, options.dataset, options.memoryConfig);
345
+ // For readOnlyConfig, use the base config's prepared data directly
346
+ const effectiveConfig =
347
+ configDef.readOnlyConfig && configDef.baseConfig ? configDef.baseConfig : options.memoryConfig;
348
+ const preparedDir = join(options.preparedDataDir || this.preparedDataDir, options.dataset, effectiveConfig);
349
+
350
+ // For readOnlyConfig, create a separate output directory for debug files
351
+ // This prevents writing to the base config's prepared data
352
+ const outputDir = configDef.readOnlyConfig
353
+ ? join(options.preparedDataDir || this.preparedDataDir, options.dataset, options.memoryConfig)
354
+ : preparedDir;
66
355
 
67
356
  if (!existsSync(preparedDir)) {
68
- throw new Error(`Prepared data not found at: ${preparedDir}\nPlease run 'longmemeval prepare' first.`);
357
+ throw new Error(`Prepared data not found at: ${preparedDir}
358
+ Please run 'longmemeval prepare' first.`);
69
359
  }
70
360
 
71
- // Load prepared questions
361
+ if (configDef.readOnlyConfig && configDef.baseConfig) {
362
+ console.log(chalk.gray(`Using prepared data from: ${configDef.baseConfig}`));
363
+ }
364
+
365
+ // Load original dataset to get correct question order
72
366
  const spinner = ora('Loading prepared data...').start();
367
+ const originalQuestions = await this.loader.loadDataset(options.dataset);
368
+ const questionIdOrder = new Map(originalQuestions.map((q, i) => [q.question_id, i]));
369
+
370
+ // Load prepared questions
73
371
  const questionDirs = await readdir(preparedDir);
74
372
  const preparedQuestions: PreparedQuestionMeta[] = [];
75
373
 
76
374
  let skippedCount = 0;
77
375
  let failedCount = 0;
376
+ let reconciledCount = 0;
78
377
  for (const questionDir of questionDirs) {
79
378
  const questionPath = join(preparedDir, questionDir);
80
379
  const metaPath = join(questionPath, 'meta.json');
81
380
  const progressPath = join(questionPath, 'progress.json');
82
381
 
83
382
  // Check if question has been prepared
84
- if (existsSync(metaPath)) {
383
+ let hasMetaJson = existsSync(metaPath);
384
+
385
+ // If no meta.json and this is a derived config (non-readOnly), try to reconcile from base
386
+ if (!hasMetaJson && configDef.baseConfig && !configDef.readOnlyConfig) {
387
+ const baseDir = join(options.preparedDataDir || this.preparedDataDir, options.dataset, configDef.baseConfig);
388
+ const baseQuestionDir = join(baseDir, questionDir);
389
+ const baseMetaPath = join(baseQuestionDir, 'meta.json');
390
+
391
+ if (existsSync(baseMetaPath)) {
392
+ spinner.text = `Reconciling ${questionDir} from ${configDef.baseConfig}...`;
393
+ const result = await reconcileQuestion({
394
+ questionId: questionDir,
395
+ targetConfig: options.memoryConfig,
396
+ preparedDataDir: options.preparedDataDir || this.preparedDataDir,
397
+ dataset: options.dataset,
398
+ model: configDef.omModel ?? undefined,
399
+ });
400
+
401
+ if (result.copied) {
402
+ reconciledCount++;
403
+ hasMetaJson = true;
404
+ }
405
+ }
406
+ }
407
+
408
+ if (hasMetaJson) {
85
409
  // Check if there's an incomplete or failed preparation
86
410
  if (existsSync(progressPath)) {
87
411
  const progress = JSON.parse(await readFile(progressPath, 'utf-8'));
@@ -100,48 +424,251 @@ export class RunCommand {
100
424
  }
101
425
  }
102
426
 
427
+ // Sort prepared questions to match original dataset order
428
+ preparedQuestions.sort((a, b) => {
429
+ const orderA = questionIdOrder.get(a.questionId) ?? Infinity;
430
+ const orderB = questionIdOrder.get(b.questionId) ?? Infinity;
431
+ return orderA - orderB;
432
+ });
433
+
434
+ const statusParts: string[] = [];
435
+ if (reconciledCount > 0) statusParts.push(`${reconciledCount} reconciled`);
436
+ if (skippedCount > 0) statusParts.push(`${skippedCount} incomplete`);
437
+ if (failedCount > 0) statusParts.push(`${failedCount} failed`);
438
+
103
439
  spinner.succeed(
104
- `Loaded ${preparedQuestions.length} prepared questions${skippedCount > 0 || failedCount > 0 ? ` (${skippedCount} incomplete, ${failedCount} failed)` : ''}`,
440
+ `Loaded ${preparedQuestions.length} prepared questions${statusParts.length > 0 ? ` (${statusParts.join(', ')})` : ''}`,
105
441
  );
106
442
 
107
443
  if (skippedCount > 0) {
108
444
  console.log(
109
445
  chalk.yellow(
110
- `\n⚠️ ${skippedCount} question${skippedCount > 1 ? 's' : ''} skipped due to incomplete preparation.`,
446
+ `
447
+ ⚠️ ${skippedCount} question${skippedCount > 1 ? 's' : ''} skipped due to incomplete preparation.`,
111
448
  ),
112
449
  );
113
- console.log(chalk.gray(` Run 'prepare' command to complete preparation.\n`));
450
+ console.log(
451
+ chalk.gray(` Run 'prepare' command to complete preparation.
452
+ `),
453
+ );
114
454
  }
115
455
 
116
456
  if (failedCount > 0) {
117
457
  console.log(
118
- chalk.red(`\n⚠️ ${failedCount} question${failedCount > 1 ? 's' : ''} skipped due to failed preparation.`),
458
+ chalk.red(`
459
+ ⚠️ ${failedCount} question${failedCount > 1 ? 's' : ''} skipped due to failed preparation.`),
460
+ );
461
+ console.log(
462
+ chalk.gray(` Check error logs and re-run 'prepare' command.
463
+ `),
119
464
  );
120
- console.log(chalk.gray(` Check error logs and re-run 'prepare' command.\n`));
121
465
  }
122
466
 
123
467
  // Filter by questionId if specified
124
468
  let questionsToProcess = preparedQuestions;
125
- if (options.questionId) {
469
+
470
+ // Handle --from-failures option
471
+ if (options.fromFailures) {
472
+ let failuresPath = typeof options.fromFailures === 'string' ? options.fromFailures : '';
473
+
474
+ // Handle flag with no value (true) or "latest" - find the most recent failures.json for this config
475
+ if (options.fromFailures === true || options.fromFailures === 'latest') {
476
+ const resultsDir = join(options.outputDir || this.outputDir, options.memoryConfig);
477
+ if (!existsSync(resultsDir)) {
478
+ throw new Error(`No results directory found for config: ${options.memoryConfig}`);
479
+ }
480
+
481
+ const runDirs = (await readdir(resultsDir))
482
+ .filter(d => d.startsWith('run_'))
483
+ .sort()
484
+ .reverse(); // Most recent first
485
+
486
+ let latestFailuresPath: string | null = null;
487
+ for (const runDir of runDirs) {
488
+ const candidatePath = join(resultsDir, runDir, 'failures.json');
489
+ if (existsSync(candidatePath)) {
490
+ latestFailuresPath = candidatePath;
491
+ break;
492
+ }
493
+ }
494
+
495
+ if (!latestFailuresPath) {
496
+ throw new Error(`No failures.json found in any run for config: ${options.memoryConfig}`);
497
+ }
498
+ failuresPath = latestFailuresPath;
499
+ console.log(chalk.gray(`Found latest failures: ${failuresPath}\n`));
500
+ }
501
+
502
+ // Load failures.json and filter to those question IDs
503
+ if (!existsSync(failuresPath)) {
504
+ throw new Error(`Failures file not found: ${failuresPath}`);
505
+ }
506
+ const fromFailuresData = JSON.parse(await readFile(failuresPath, 'utf-8')) as FailuresFile;
507
+ const failedIds = new Set(fromFailuresData.questionIds);
508
+ questionsToProcess = preparedQuestions.filter(q => failedIds.has(q.questionId));
509
+
510
+ if (questionsToProcess.length === 0) {
511
+ throw new Error(`No matching questions found for ${fromFailuresData.questionIds.length} failed IDs`);
512
+ }
513
+
514
+ // Filter by --older-than if specified (check meta.json mtime)
515
+ let skippedRecentCount = 0;
516
+ if (options.olderThan) {
517
+ const maxAgeMs = parseDuration(options.olderThan);
518
+ const cutoffTime = Date.now() - maxAgeMs;
519
+
520
+ const filteredQuestions: PreparedQuestionMeta[] = [];
521
+ for (const q of questionsToProcess) {
522
+ const metaPath = join(preparedDir, q.questionId, 'meta.json');
523
+ if (existsSync(metaPath)) {
524
+ const metaStat = await stat(metaPath);
525
+ if (metaStat.mtimeMs > cutoffTime) {
526
+ // Recently prepared, skip
527
+ skippedRecentCount++;
528
+ continue;
529
+ }
530
+ }
531
+ filteredQuestions.push(q);
532
+ }
533
+ questionsToProcess = filteredQuestions;
534
+
535
+ if (questionsToProcess.length === 0) {
536
+ console.log(
537
+ chalk.green(
538
+ `\n✓ All ${skippedRecentCount} failed questions were recently prepared (within ${options.olderThan})`,
539
+ ),
540
+ );
541
+ console.log(chalk.gray(` Nothing to re-run.\n`));
542
+ // Return empty metrics
543
+ return {
544
+ total_questions: 0,
545
+ correct_answers: 0,
546
+ overall_accuracy: 0,
547
+ accuracy_by_type: {},
548
+ abstention_accuracy: 0,
549
+ };
550
+ }
551
+ }
552
+
553
+ console.log(
554
+ chalk.yellow(`\n🔄 Running ${questionsToProcess.length} failed questions from: ${options.fromFailures}`),
555
+ );
556
+ console.log(chalk.gray(` Run ID: ${fromFailuresData.runId}`));
557
+ console.log(
558
+ chalk.gray(` Original failures: ${fromFailuresData.totalFailed}/${fromFailuresData.totalQuestions}`),
559
+ );
560
+ if (skippedRecentCount > 0) {
561
+ console.log(chalk.gray(` Skipped ${skippedRecentCount} recently prepared (within ${options.olderThan})`));
562
+ }
563
+ console.log();
564
+ } else if (options.questionId) {
126
565
  questionsToProcess = preparedQuestions.filter(q => q.questionId === options.questionId);
127
566
  if (questionsToProcess.length === 0) {
128
567
  throw new Error(`Question with ID "${options.questionId}" not found in prepared data`);
129
568
  }
130
- console.log(chalk.yellow(`\nFocusing on question: ${options.questionId}\n`));
131
- } else if (options.subset) {
132
- // Apply subset if requested
133
- questionsToProcess = preparedQuestions.slice(0, options.subset);
134
569
  console.log(
135
- chalk.gray(`\nApplying subset: ${options.subset} questions from ${preparedQuestions.length} total\n`),
570
+ chalk.yellow(`
571
+ Focusing on question: ${options.questionId}
572
+ `),
136
573
  );
574
+ } else {
575
+ // Filter by question type(s) if specified (supports comma-separated)
576
+ if (options.questionType) {
577
+ const requestedTypes = options.questionType.split(',').map(t => t.trim());
578
+ const availableTypes = [...new Set(preparedQuestions.map(q => q.questionType))].sort();
579
+
580
+ // Validate all requested types exist
581
+ const invalidTypes = requestedTypes.filter(t => !availableTypes.includes(t));
582
+ if (invalidTypes.length > 0) {
583
+ throw new Error(
584
+ `Invalid question type(s): ${invalidTypes.join(', ')}. Available types: ${availableTypes.join(', ')}`,
585
+ );
586
+ }
587
+
588
+ questionsToProcess = preparedQuestions.filter(q => requestedTypes.includes(q.questionType));
589
+ if (questionsToProcess.length === 0) {
590
+ throw new Error(
591
+ `No questions found with type(s) "${options.questionType}". Available types: ${availableTypes.join(', ')}`,
592
+ );
593
+ }
594
+ console.log(
595
+ chalk.yellow(`
596
+ Filtering to question type${requestedTypes.length > 1 ? 's' : ''}: ${requestedTypes.join(', ')} (${questionsToProcess.length} questions)
597
+ `),
598
+ );
599
+ }
600
+ // Apply stratified sampling if perTypeCount is set (only if not filtering by type)
601
+ if (options.perTypeCount && !options.questionType) {
602
+ console.log(
603
+ chalk.gray(`
604
+ Applying stratified sampling (${options.perTypeCount} per type):`),
605
+ );
606
+ // Map prepared questions to include question_type for sampling
607
+ const withTypes = preparedQuestions.map(q => ({
608
+ ...q,
609
+ question_id: q.questionId,
610
+ question_type: q.questionType,
611
+ }));
612
+ const sampled = applyStratifiedSampling(withTypes, options.perTypeCount);
613
+ questionsToProcess = sampled.map(q => {
614
+ const { question_id, question_type, ...rest } = q;
615
+ return rest as (typeof preparedQuestions)[0];
616
+ });
617
+ }
618
+
619
+ // Apply comb sampling if combSampleSize is set (works with -t type filter)
620
+ if (options.combSampleSize) {
621
+ const combOffset = options.combOffset ?? 10;
622
+ const startOffset = options.combStartOffset ?? 0;
623
+ console.log(
624
+ chalk.gray(`
625
+ Applying comb sampling (${options.combSampleSize} per type, offset=${combOffset}, start=${startOffset}):`),
626
+ );
627
+ // Map prepared questions to include question_type for sampling
628
+ const withTypes = questionsToProcess.map(q => ({
629
+ ...q,
630
+ question_id: q.questionId,
631
+ question_type: q.questionType,
632
+ }));
633
+ const sampled = applyCombSampling(withTypes, options.combSampleSize, combOffset, startOffset);
634
+ questionsToProcess = sampled.map(q => {
635
+ const { question_id, question_type, ...rest } = q;
636
+ return rest as (typeof preparedQuestions)[0];
637
+ });
638
+ }
639
+
640
+ // Apply offset and subset
641
+ const totalBeforeSlice = questionsToProcess.length;
642
+ const offset = options.offset || 0;
643
+ if (offset > 0) {
644
+ questionsToProcess = questionsToProcess.slice(offset);
645
+ }
646
+ if (options.subset) {
647
+ questionsToProcess = questionsToProcess.slice(0, options.subset);
648
+ }
649
+ if (offset > 0 || options.subset) {
650
+ console.log(
651
+ chalk.gray(`Processing questions ${offset + 1}-${offset + questionsToProcess.length} of ${totalBeforeSlice}${options.questionType ? ` ${options.questionType}` : ''} total
652
+ `),
653
+ );
654
+ }
137
655
  }
138
656
 
139
657
  console.log(
140
- chalk.yellow(`\nEvaluating ${questionsToProcess.length} question${questionsToProcess.length !== 1 ? 's' : ''}\n`),
658
+ chalk.yellow(`
659
+ Evaluating ${questionsToProcess.length} question${questionsToProcess.length !== 1 ? 's' : ''}
660
+ `),
141
661
  );
142
662
 
663
+ // Filter out already-completed questions (for resume)
664
+ const remainingQuestions = questionsToProcess.filter(q => !completedQuestionIds.has(q.questionId));
665
+ if (options.resume && remainingQuestions.length < questionsToProcess.length) {
666
+ console.log(
667
+ chalk.yellow(`Skipping ${questionsToProcess.length - remainingQuestions.length} already-completed questions`),
668
+ );
669
+ }
670
+
143
671
  // Process questions with concurrency control
144
- const results: EvaluationResult[] = [];
145
672
  const concurrency = options.concurrency || 5;
146
673
  const questionSpinner = ora('Evaluating questions...').start();
147
674
 
@@ -153,16 +680,19 @@ export class RunCommand {
153
680
  const activeEvaluations = new Map<number, { questionId: string; status: string }>();
154
681
 
155
682
  // Function to update progress display
683
+ const totalToProcess = remainingQuestions.length;
156
684
  let lastText = '';
157
685
  const updateProgress = () => {
158
686
  const elapsed = Math.round((Date.now() - startTime) / 1000);
159
687
  const rate = elapsed > 0 ? completedCount / elapsed : 0;
160
- const remaining = rate > 0 ? Math.round((questionsToProcess.length - completedCount) / rate) : 0;
688
+ const remaining = rate > 0 ? Math.round((totalToProcess - completedCount) / rate) : 0;
161
689
 
162
- let progressText = `Overall: ${completedCount}/${questionsToProcess.length} (${inProgressCount} in progress, ${Math.round(rate * 60)} q/min, ~${remaining}s remaining)`;
690
+ let progressText = `Overall: ${completedCount}/${totalToProcess} (${inProgressCount} in progress, ${Math.round(rate * 60)} q/min, ~${remaining}s remaining)`;
163
691
 
164
692
  if (activeEvaluations.size > 0 && concurrency > 1) {
165
- progressText += '\n\nActive evaluations:';
693
+ progressText += `
694
+
695
+ Active evaluations:`;
166
696
 
167
697
  // Sort active evaluations by completion status
168
698
  const sortedEvaluations = Array.from(activeEvaluations.entries())
@@ -180,7 +710,8 @@ export class RunCommand {
180
710
 
181
711
  sortedEvaluations.forEach(({ index, info, progress }) => {
182
712
  const percentage = (progress * 100).toFixed(0);
183
- progressText += `\n [${index + 1}] ${info.questionId} - ${info.status} (${percentage}%)`;
713
+ progressText += `
714
+ [${index + 1}] ${info.questionId} - ${info.status} (${percentage}%)`;
184
715
  });
185
716
  }
186
717
 
@@ -191,12 +722,10 @@ export class RunCommand {
191
722
  };
192
723
 
193
724
  // Create a queue of questions to evaluate
194
- const questionQueue = [...questionsToProcess];
725
+ const questionQueue = [...remainingQuestions];
195
726
 
196
727
  // Function to process next question from queue
197
- const processNextQuestion = async (slotIndex: number): Promise<EvaluationResult[]> => {
198
- const workerResults: EvaluationResult[] = [];
199
-
728
+ const processNextQuestion = async (slotIndex: number): Promise<void> => {
200
729
  while (questionQueue.length > 0) {
201
730
  const meta = questionQueue.shift();
202
731
  if (!meta) break;
@@ -208,8 +737,10 @@ export class RunCommand {
208
737
  const result = await this.evaluateQuestion(
209
738
  meta,
210
739
  preparedDir,
211
- retry4o.model,
740
+ outputDir,
741
+ normalizeModelId(configDef.agentModel ?? 'gpt-4o'),
212
742
  options,
743
+ configDef,
213
744
  concurrency > 1
214
745
  ? {
215
746
  updateStatus: (status: string) => {
@@ -240,15 +771,25 @@ export class RunCommand {
240
771
  console.log(chalk.yellow(` Expected: "${meta.answer}"`));
241
772
  }
242
773
 
774
+ // Show improved result if applicable
775
+ if (result.improved_question) {
776
+ console.log(
777
+ chalk.cyan(` ↳ improved:`),
778
+ chalk[result.improved_is_correct ? 'green' : 'red'](`${result.improved_is_correct ? '✓' : '✗'}`),
779
+ );
780
+ // if (!result.improved_is_correct) {
781
+ console.log(chalk.gray(` Q: "${result.improved_question}"`));
782
+ console.log(chalk.gray(` A: "${result.improved_hypothesis}"`));
783
+ // }
784
+ }
785
+
243
786
  // Re-render the spinner
244
787
  questionSpinner.render();
245
788
  }
246
789
 
247
- // Don't update progress here - let the periodic timer handle it
248
- workerResults.push(result);
790
+ // Append result to JSONL file immediately (crash-safe incremental save)
791
+ await appendFile(resultsPath, JSON.stringify(result) + '\n');
249
792
  }
250
-
251
- return workerResults;
252
793
  };
253
794
 
254
795
  // Set up periodic progress updates
@@ -257,28 +798,38 @@ export class RunCommand {
257
798
  // Create worker slots
258
799
  const workers = Array.from({ length: concurrency }, (_, i) => processNextQuestion(i));
259
800
 
260
- // Wait for all workers to complete and collect results
261
- const workerResults = await Promise.all(workers);
262
-
263
- // Process results from all workers
264
- for (const workerResultArray of workerResults) {
265
- results.push(...workerResultArray);
266
- }
801
+ // Wait for all workers to complete
802
+ await Promise.all(workers);
267
803
 
268
804
  // Clear the interval
269
805
  clearInterval(progressInterval);
270
806
 
271
- questionSpinner.succeed(`Evaluated ${results.length} questions`);
807
+ questionSpinner.succeed(`Evaluated ${totalToProcess} questions`);
808
+
809
+ // Load all results from JSONL file (includes existing + new)
810
+ const allResultsContent = await readFile(resultsPath, 'utf-8');
811
+ const allResults = allResultsContent
812
+ .trim()
813
+ .split('\n')
814
+ .filter(line => line.trim())
815
+ .map(line => JSON.parse(line) as EvaluationResult);
272
816
 
273
817
  // Calculate metrics
274
- console.log(chalk.blue('\n📊 Calculating metrics...\n'));
275
- const metrics = this.calculateMetrics(results);
818
+ console.log(
819
+ chalk.blue(`
820
+ 📊 Calculating metrics...
821
+ `),
822
+ );
823
+ const metrics = this.calculateMetrics(allResults);
276
824
 
277
- // Save results
278
- await this.saveResults(runDir, results, metrics, options);
825
+ // Save final metrics and failures (results.jsonl already saved incrementally)
826
+ await this.saveFinalResults(runDir, allResults, metrics, options);
279
827
 
280
- // Display results
281
- this.displayMetrics(metrics, options);
828
+ // Display uninvestigated failures first (questions for investigation)
829
+ this.displayUninvestigatedFailures(allResults);
830
+
831
+ // Display results summary at the end
832
+ this.displayMetrics(metrics, options, configDef);
282
833
 
283
834
  return metrics;
284
835
  }
@@ -286,8 +837,10 @@ export class RunCommand {
286
837
  private async evaluateQuestion(
287
838
  meta: PreparedQuestionMeta,
288
839
  preparedDir: string,
289
- modelProvider: any,
840
+ outputDir: string,
841
+ agentModelId: `${string}/${string}`,
290
842
  options: RunOptions,
843
+ configDef: MemoryConfigDefinition,
291
844
  spinner?: Ora | { updateStatus: (status: string) => void },
292
845
  ): Promise<EvaluationResult> {
293
846
  const questionStart = Date.now();
@@ -305,11 +858,20 @@ export class RunCommand {
305
858
 
306
859
  // Load the prepared storage and vector store
307
860
  const questionDir = join(preparedDir, meta.questionId);
308
- const benchmarkStore = new BenchmarkStore('read');
861
+ // Separate output directory for debug files (prevents writing to base config for readOnlyConfig)
862
+ const questionOutputDir = join(outputDir, meta.questionId);
309
863
  const benchmarkVectorStore = new BenchmarkVectorStore('read');
310
864
 
311
- await benchmarkStore.init();
312
- await benchmarkStore.hydrate(join(questionDir, 'db.json'));
865
+ const memoryOptions = getMemoryOptions(options.memoryConfig);
866
+ const usesObservationalMemory = configDef.usesObservationalMemory;
867
+
868
+ // Only load BenchmarkStore for non-OM configs (OM uses PersistableInMemoryMemory)
869
+ let benchmarkStore: BenchmarkStore | undefined;
870
+ if (!usesObservationalMemory) {
871
+ benchmarkStore = new BenchmarkStore('read');
872
+ await benchmarkStore.init();
873
+ await benchmarkStore.hydrate(join(questionDir, 'db.json'));
874
+ }
313
875
 
314
876
  // Hydrate vector store if it exists
315
877
  const vectorPath = join(questionDir, 'vector.json');
@@ -318,45 +880,237 @@ export class RunCommand {
318
880
  updateStatus(`Loading vector embeddings for ${meta.questionId}...`);
319
881
  }
320
882
 
321
- const memoryOptions = getMemoryOptions(options.memoryConfig);
883
+ // Create memory with the hydrated stores (for non-OM configs)
884
+ // Note: BenchmarkStore is outdated and doesn't fully implement MastraStorage
885
+ // Using 'as any' as a workaround since OM configs use PersistableInMemoryMemory instead
886
+ const memory = usesObservationalMemory
887
+ ? undefined
888
+ : new Memory({
889
+ storage: benchmarkStore as any,
890
+ vector: benchmarkVectorStore,
891
+ embedder: cachedOpenAI.embedding('text-embedding-3-small'),
892
+ options: memoryOptions.options,
893
+ });
322
894
 
323
- // Create memory with the hydrated stores
324
- const memory = new Memory({
325
- storage: benchmarkStore,
326
- vector: benchmarkVectorStore,
327
- embedder: cachedOpenAI.embedding('text-embedding-3-small'),
328
- options: memoryOptions.options,
329
- });
895
+ // Create observational memory processor if using OM config
896
+ let observationalMemory: ObservationalMemory | undefined;
897
+ let omStorage: PersistableInMemoryMemory | undefined;
898
+
899
+ if (usesObservationalMemory) {
900
+ // Use PersistableInMemoryMemory for ObservationalMemory
901
+ omStorage = new PersistableInMemoryMemory({ readOnly: true });
902
+
903
+ // Hydrate OM storage from prepared data
904
+ const omPath = join(questionDir, 'om.json');
905
+ if (existsSync(omPath)) {
906
+ await omStorage.hydrate(omPath);
907
+ updateStatus(`Loaded OM data for ${meta.questionId}...`);
908
+ }
909
+
910
+ observationalMemory = new ObservationalMemory({
911
+ obscureThreadIds: true, // can't show answer_x in context when we put the thread id in xml tags
912
+ storage: omStorage,
913
+ observation: {
914
+ // model: retry4o.model,
915
+ messageTokens: OBSERVATIONAL_MEMORY_DEFAULTS.observation.messageTokens,
916
+ recognizePatterns: false,
917
+ },
918
+ reflection: {
919
+ // model: retry4o.model,
920
+ observationTokens: OBSERVATIONAL_MEMORY_DEFAULTS.reflection.observationTokens,
921
+ recognizePatterns: false,
922
+ },
923
+ scope: 'resource',
924
+ });
925
+ }
926
+
927
+ // Create observation semantic filter if using RAG config
928
+ let observationRagFilter: ObservationSemanticFilter | undefined;
929
+ const usesObservationRag = configDef.usesObservationRag;
930
+
931
+ if (usesObservationRag && usesObservationalMemory) {
932
+ // Use a shared cache directory for embeddings (content-based, reusable across configs)
933
+ // Subdirectory by model name for future multi-model support
934
+ const embeddingsCacheDir = join(preparedDir, '..', '.embeddings-cache', 'fastembed-small');
935
+ observationRagFilter = new ObservationSemanticFilter({
936
+ embedder: fastembed.small,
937
+ topK: configDef.ragTopK ?? 50,
938
+ // minSimilarity: 0.4,
939
+ includeCurrentTask: false,
940
+ includeSuggestedResponse: false,
941
+ includePatterns: false,
942
+ cacheDir: embeddingsCacheDir,
943
+ preferenceBoost: configDef.ragPreferenceBoost ?? false,
944
+ });
945
+ }
330
946
 
331
947
  // Create agent with the specified model
332
948
  const agentInstructions = `You are a helpful assistant with access to extensive conversation history.
333
- When answering questions, carefully review the conversation history to identify and use any relevant user preferences, interests, or specific details they have mentioned.
334
- For example, if the user previously mentioned they prefer a specific software, tool, or approach, tailor your recommendations to match their stated preferences.
335
- Be specific rather than generic when the user has expressed clear preferences in past conversations. If there is a clear preference, focus in on that, and do not add additional irrelevant information.`;
949
+ When answering questions, carefully review the conversation history to identify and use any relevant user preferences, interests, or specific details they have mentioned.`;
950
+
951
+ // Ensure output directory exists for readOnlyConfig
952
+ if (!existsSync(questionOutputDir)) {
953
+ mkdirSync(questionOutputDir, { recursive: true });
954
+ }
955
+ const omDebugPath = join(questionOutputDir, 'om.md');
956
+
957
+ // Create date injector processor if we have a question date
958
+ // This injects the date into the user message in official LongMemEval format:
959
+ // "Current Date: {date}\nQuestion: {question}"
960
+ const dateInjector = meta.questionDate ? new DateInjector({ date: meta.questionDate }) : null;
336
961
 
337
962
  const agent = new Agent({
338
- name: 'longmemeval-agent',
339
- model: modelProvider,
340
- instructions: agentInstructions,
963
+ id: 'longmemeval-agent',
964
+ name: 'LongMemEval Agent',
965
+ model: agentModelId,
966
+ instructions: [
967
+ { role: 'system', content: agentInstructions },
968
+ // prevent openai prompt caching
969
+ {
970
+ role: 'system',
971
+ content: `
972
+ cache: ${Math.random()}`,
973
+ },
974
+ ],
975
+ // tools: observationalMemory
976
+ // ? {
977
+ // recall: observationalMemory?.getRecallTool(),
978
+ // }
979
+ // : undefined,
341
980
  memory,
981
+ // For OM, use processors instead of memory
982
+ // OM handles message loading itself via cursor-based loadUnobservedMessages
983
+ // MessageHistory must come first in output to save messages before OM observes them
984
+ inputProcessors: usesObservationalMemory
985
+ ? [
986
+ observationalMemory!,
987
+ // Add RAG filter after OM if enabled - it will filter OM's injected observations
988
+ ...(observationRagFilter ? [observationRagFilter] : []),
989
+ // Inject date into user message in official LongMemEval format
990
+ // Placed BEFORE RAG so the date is part of the RAG query (more authentic to benchmark)
991
+ // Can be moved after RAG to experiment with ordering
992
+ ...(dateInjector ? [dateInjector] : []),
993
+ {
994
+ id: 'debug',
995
+ processInputStep: args => {
996
+ // Check tagged OM messages first, then fall back to all system messages
997
+ // (RAG filter replaces tagged messages with untagged ones)
998
+ let omm = args.messageList.getSystemMessages(`observational-memory`);
999
+ const taggedCount = omm.length;
1000
+ if (!omm.length) {
1001
+ // RAG filter may have replaced tagged messages - get all system messages
1002
+ omm = args.messageList.getAllSystemMessages();
1003
+ }
1004
+ // Find the message with observations (contains <observations> tag)
1005
+ const observationsMsg = omm.find(m => {
1006
+ const content = typeof m.content === 'string' ? m.content : '';
1007
+ return content.includes('<observations>');
1008
+ });
1009
+ const msgToWrite = observationsMsg || omm[0];
1010
+ if (msgToWrite?.content) {
1011
+ const content =
1012
+ typeof msgToWrite.content === 'string' ? msgToWrite.content : JSON.stringify(msgToWrite.content);
1013
+ writeFileSync(
1014
+ omDebugPath,
1015
+ `[Debug: tagged=${taggedCount}, total=${omm.length}, hasObservations=${!!observationsMsg}]
1016
+
1017
+ ` +
1018
+ content +
1019
+ `
1020
+
1021
+ ${JSON.stringify(args.messageList.get.all.core(), null, 2)}
1022
+
1023
+ ${JSON.stringify(args.requestContext?.get('MastraMemory') || {}, null, 2)}`,
1024
+ );
1025
+ }
1026
+ return args.messageList;
1027
+ },
1028
+ },
1029
+ ]
1030
+ : undefined,
1031
+ outputProcessors: usesObservationalMemory
1032
+ ? [
1033
+ {
1034
+ id: 'debug-output',
1035
+ processOutputResult: args => {
1036
+ const responses = args.messageList.get.response.v1();
1037
+ if (existsSync(omDebugPath)) {
1038
+ appendFileSync(
1039
+ omDebugPath,
1040
+ `
1041
+ ${JSON.stringify(responses, null, 2)}`,
1042
+ );
1043
+ }
1044
+ return args.messageList;
1045
+ },
1046
+ },
1047
+ ]
1048
+ : undefined,
342
1049
  });
343
1050
 
344
1051
  // Create a fresh thread for the evaluation question
345
1052
  const evalThreadId = `eval_${meta.questionId}_${Date.now()}`;
346
1053
 
1054
+ // Parse questionDate for relative time annotations in OM context
1055
+ // Format: "2023/05/30 (Tue) 10:18"
1056
+ let questionDate: Date | undefined;
1057
+ if (meta.questionDate) {
1058
+ const match = meta.questionDate.match(/^(\d{4})\/(\d{2})\/(\d{2}).*?(\d{2}):(\d{2})/);
1059
+ if (match) {
1060
+ const [, year, month, day, hour, minute] = match;
1061
+ questionDate = new Date(
1062
+ parseInt(year),
1063
+ parseInt(month) - 1, // JS months are 0-indexed
1064
+ parseInt(day),
1065
+ parseInt(hour),
1066
+ parseInt(minute),
1067
+ );
1068
+ }
1069
+ }
1070
+
1071
+ // Create request context with currentDate for OM relative time annotations
1072
+ // Also set MastraMemory context so ObservationalMemory processor can find threadId/resourceId
1073
+ const requestContext = new RequestContext([
1074
+ ['currentDate', questionDate ?? new Date()],
1075
+ ['MastraMemory', { thread: { id: evalThreadId }, resourceId: meta.resourceId }],
1076
+ ]);
1077
+
347
1078
  updateStatus(`${meta.threadIds.length} sessions, ${options.memoryConfig}`);
348
1079
 
349
- const response = await agent.generate(meta.question, {
350
- threadId: evalThreadId,
351
- resourceId: meta.resourceId,
352
- temperature: 0,
353
- context: meta.questionDate ? [{ role: 'system', content: `Todays date is ${meta.questionDate}` }] : undefined,
354
- });
1080
+ let response = await withRateLimitRetry(
1081
+ () =>
1082
+ agent.generate(meta.question, {
1083
+ threadId: evalThreadId,
1084
+ resourceId: meta.resourceId,
1085
+ requestContext,
1086
+ modelSettings: {
1087
+ temperature: 0,
1088
+ },
1089
+ context: [],
1090
+ }),
1091
+ spinner,
1092
+ );
1093
+
1094
+ // Track token usage from the main question
1095
+ let usage = response.totalUsage ?? response.usage;
1096
+
1097
+ console.log(
1098
+ response.text +
1099
+ `
1100
+
1101
+ `,
1102
+ );
355
1103
 
1104
+ const evalModelId = normalizeModelId(configDef.evalModel ?? 'openai/gpt-4o');
356
1105
  const evalAgent = new Agent({
357
- name: 'longmemeval-metric-agent',
358
- model: retry4o.model,
359
- instructions: 'You are an evaluation assistant. Answer questions precisely and concisely.',
1106
+ id: 'longmemeval-metric-agent',
1107
+ name: 'LongMemEval Metric Agent',
1108
+ model: evalModelId,
1109
+ // Official LongMemEval uses no system prompt for the judge - just the per-question-type
1110
+ // prompts sent as user messages. Keeping empty to match official methodology.
1111
+ // Old instructions (non-standard):
1112
+ // 'You are an evaluation assistant. Answer questions precisely and concisely. Any answer to a question you see where the answer contains "I dont know", but the answer is also stated, is correct. Give leeway for conversion units. If the answer is 1.5 hours, but the given answer is 90 minutes, that is also correct. If the response contains the answer plus additional information, that is correct too.',
1113
+ instructions: '',
360
1114
  });
361
1115
 
362
1116
  const metric = new LongMemEvalMetric({
@@ -370,17 +1124,145 @@ Be specific rather than generic when the user has expressed clear preferences in
370
1124
  answer: meta.answer,
371
1125
  });
372
1126
 
373
- const result = await metric.measure(input, response.text);
374
- const isCorrect = result.score === 1;
1127
+ const result = await withRateLimitRetry(() => metric.measure(input, response.text), spinner);
1128
+ let isCorrect = result.score === 1;
1129
+
1130
+ // Check if there's an improved version - if so, we'll only retry that one
1131
+ const hasImprovedVersion = !!(meta.improvedQuestion || meta.improvedAnswer);
1132
+
1133
+ // Retry failed evaluations: always at least 1 retry, up to 5 if requiresRetry is set
1134
+ // Only retry vanilla if there's NO improved version (otherwise we retry the improved one)
1135
+ let retryCount = 0;
1136
+ const maxRetries = meta.requiresRetry || hasImprovedVersion ? 2 : 0;
1137
+ while (!isCorrect && !hasImprovedVersion && retryCount < maxRetries) {
1138
+ retryCount++;
1139
+ updateStatus(`Retry ${retryCount}/${maxRetries} for ${meta.questionId}...`);
1140
+
1141
+ const retryThreadId = `eval_retry_${meta.questionId}_${retryCount}_${Date.now()}`;
1142
+ const retryResponse = await withRateLimitRetry(
1143
+ () =>
1144
+ agent.generate(meta.question, {
1145
+ threadId: retryThreadId,
1146
+ resourceId: meta.resourceId,
1147
+ requestContext,
1148
+ modelSettings: {
1149
+ temperature: 0,
1150
+ },
1151
+ }),
1152
+ spinner,
1153
+ );
1154
+
1155
+ const retryResult = await withRateLimitRetry(() => metric.measure(input, retryResponse.text), spinner);
1156
+ if (retryResult.score === 1) {
1157
+ isCorrect = true;
1158
+ // Update response to the successful retry for logging
1159
+ response = retryResponse;
1160
+ // Update usage to the successful retry
1161
+ usage = retryResponse.totalUsage ?? retryResponse.usage;
1162
+ }
1163
+ }
1164
+ const didRetry = retryCount > 0;
1165
+
1166
+ // Run improved evaluation if improved question OR improved answer exists (unless --no-fixed)
1167
+ let improvedQuestion: string | undefined;
1168
+ let improvedAnswer: string | undefined;
1169
+ let improvedHypothesis: string | undefined;
1170
+ let improvedIsCorrect: boolean | undefined;
1171
+ let improvedUsage: typeof usage | undefined;
1172
+
1173
+ if (!options.skipFixed) {
1174
+ // Normalize: if only improvedAnswer exists, copy the original question to improvedQuestion
1175
+ // This simplifies the logic - we always run a "fixed" evaluation if either field is set
1176
+ improvedQuestion = meta.improvedQuestion ?? (meta.improvedAnswer ? meta.question : undefined);
1177
+ improvedAnswer = meta.improvedAnswer ?? meta.answer;
1178
+ }
1179
+
1180
+ if (improvedQuestion) {
1181
+ // If the improved question is the same as the original (only answer changed),
1182
+ // reuse the vanilla response. Otherwise, run a new query.
1183
+ if (improvedQuestion === meta.question) {
1184
+ improvedHypothesis = response.text;
1185
+ } else {
1186
+ updateStatus(`Running improved question for ${meta.questionId}...`);
1187
+
1188
+ // Create a separate thread for the improved question evaluation
1189
+ const improvedThreadId = `eval_improved_${meta.questionId}_${Date.now()}`;
1190
+
1191
+ const improvedResponse = await withRateLimitRetry(
1192
+ () =>
1193
+ agent.generate(improvedQuestion, {
1194
+ threadId: improvedThreadId,
1195
+ resourceId: meta.resourceId,
1196
+ requestContext,
1197
+ modelSettings: {
1198
+ temperature: 0,
1199
+ },
1200
+ }),
1201
+ spinner,
1202
+ );
1203
+
1204
+ improvedHypothesis = improvedResponse.text;
1205
+ improvedUsage = improvedResponse.totalUsage ?? improvedResponse.usage;
1206
+ }
1207
+
1208
+ const improvedInput = JSON.stringify({
1209
+ question: improvedQuestion,
1210
+ answer: improvedAnswer,
1211
+ });
1212
+
1213
+ const improvedResult = await withRateLimitRetry(
1214
+ () => metric.measure(improvedInput, improvedHypothesis!),
1215
+ spinner,
1216
+ );
1217
+ improvedIsCorrect = improvedResult.score === 1;
1218
+
1219
+ // Retry improved version: always retry at least once, up to 5 times if requiresRetry is set
1220
+ const improvedMaxRetries = meta.requiresRetry ? maxRetries : 1;
1221
+ let improvedRetryCount = 0;
1222
+ while (!improvedIsCorrect && improvedRetryCount < improvedMaxRetries) {
1223
+ improvedRetryCount++;
1224
+ updateStatus(`Retry improved ${improvedRetryCount}/${improvedMaxRetries} for ${meta.questionId}...`);
1225
+
1226
+ const retryThreadId = `eval_improved_retry_${meta.questionId}_${improvedRetryCount}_${Date.now()}`;
1227
+ const retryResponse = await withRateLimitRetry(
1228
+ () =>
1229
+ agent.generate(improvedQuestion, {
1230
+ threadId: retryThreadId,
1231
+ resourceId: meta.resourceId,
1232
+ requestContext,
1233
+ modelSettings: {
1234
+ temperature: 0,
1235
+ },
1236
+ // Date is injected via DateInjector processor in official LongMemEval format
1237
+ }),
1238
+ spinner,
1239
+ );
1240
+
1241
+ const retryResult = await withRateLimitRetry(() => metric.measure(improvedInput, retryResponse.text), spinner);
1242
+ if (retryResult.score === 1) {
1243
+ improvedIsCorrect = true;
1244
+ improvedHypothesis = retryResponse.text;
1245
+ // Update improved usage to the successful retry
1246
+ improvedUsage = retryResponse.totalUsage ?? retryResponse.usage;
1247
+ }
1248
+ }
1249
+ }
375
1250
 
376
1251
  const elapsed = ((Date.now() - questionStart) / 1000).toFixed(1);
377
1252
 
378
1253
  const isOraSpinner = spinner && 'clear' in spinner;
379
1254
  if (isOraSpinner) {
1255
+ // Show vanilla result (with retry indicator if applicable)
1256
+ const retryIndicator = didRetry
1257
+ ? isCorrect
1258
+ ? chalk.yellow(` (retry ${retryCount}/${maxRetries} ✓)`)
1259
+ : chalk.gray(` (retry ${retryCount}/${maxRetries} ✗)`)
1260
+ : '';
380
1261
  console.log(
381
1262
  chalk.blue(`▶ ${meta.questionId}`),
382
1263
  chalk.gray(`(${meta.questionType})`),
383
1264
  chalk[isCorrect ? 'green' : 'red'](`${isCorrect ? '✓' : '✗'}`),
1265
+ retryIndicator,
384
1266
  chalk.gray(`${elapsed}s`),
385
1267
  );
386
1268
  if (!isCorrect) {
@@ -388,42 +1270,120 @@ Be specific rather than generic when the user has expressed clear preferences in
388
1270
  console.log(chalk.gray(` A: "${response.text}"`));
389
1271
  console.log(chalk.yellow(` Expected: "${meta.answer}"`));
390
1272
  }
1273
+
1274
+ // Show improved result if applicable
1275
+ if (improvedQuestion) {
1276
+ // Show whether it's an improved question or just improved answer
1277
+ const label = meta.improvedQuestion ? 'improved Q' : 'improved A';
1278
+ console.log(
1279
+ chalk.cyan(` ↳ ${label}:`),
1280
+ chalk[improvedIsCorrect ? 'green' : 'red'](`${improvedIsCorrect ? '✓' : '✗'}`),
1281
+ );
1282
+ console.log(chalk.gray(` Q: "${improvedQuestion}"`));
1283
+ console.log(chalk.gray(` A: "${improvedHypothesis}"`));
1284
+ if (!improvedIsCorrect) {
1285
+ console.log(chalk.yellow(` Expected: "${improvedAnswer}"`));
1286
+ }
1287
+ }
391
1288
  }
392
1289
 
1290
+ // Track when improved version performs worse than original
1291
+ const improvedRegression = isCorrect && improvedQuestion !== undefined && !improvedIsCorrect;
1292
+
393
1293
  return {
394
1294
  question_id: meta.questionId,
1295
+ question: meta.question,
1296
+ expected_answer: meta.answer,
395
1297
  hypothesis: response.text,
396
1298
  question_type: meta.questionType as QuestionType,
397
1299
  is_correct: isCorrect,
1300
+ improved_question: improvedQuestion,
1301
+ improved_hypothesis: improvedHypothesis,
1302
+ improved_is_correct: improvedIsCorrect,
1303
+ has_improvement_info: !!(meta.improvedQuestion || meta.improvedAnswer || meta.improvementNote),
1304
+ improved_regression: improvedRegression,
1305
+ usage:
1306
+ usage && usage.inputTokens !== undefined
1307
+ ? {
1308
+ inputTokens: usage.inputTokens,
1309
+ outputTokens: usage.outputTokens ?? 0,
1310
+ totalTokens: usage.totalTokens ?? usage.inputTokens + (usage.outputTokens ?? 0),
1311
+ }
1312
+ : undefined,
1313
+ improved_usage:
1314
+ improvedUsage && improvedUsage.inputTokens !== undefined
1315
+ ? {
1316
+ inputTokens: improvedUsage.inputTokens,
1317
+ outputTokens: improvedUsage.outputTokens ?? 0,
1318
+ totalTokens: improvedUsage.totalTokens ?? improvedUsage.inputTokens + (improvedUsage.outputTokens ?? 0),
1319
+ }
1320
+ : undefined,
398
1321
  };
399
1322
  }
400
1323
 
401
- private async saveResults(
1324
+ private async saveFinalResults(
402
1325
  runDir: string,
403
1326
  results: EvaluationResult[],
404
1327
  metrics: BenchmarkMetrics,
405
1328
  options: RunOptions,
406
1329
  ): Promise<void> {
407
- // Save raw results
408
- const resultsPath = join(runDir, 'results.jsonl');
409
- const resultsContent = results.map(r => JSON.stringify(r)).join('\n');
410
- await writeFile(resultsPath, resultsContent);
1330
+ // Note: results.jsonl is already saved incrementally during the run
1331
+
1332
+ // Save failures.json for re-preparation
1333
+ // Only include questions where BOTH original and improved failed (or no improved version exists)
1334
+ const failedQuestionIds = results
1335
+ .filter(r => {
1336
+ // If original passed, not a failure
1337
+ if (r.is_correct) return false;
1338
+ // If there's an improved version and it passed, not a failure
1339
+ if (r.improved_question !== undefined && r.improved_is_correct) return false;
1340
+ // Both failed (or no improved version)
1341
+ return true;
1342
+ })
1343
+ .map(r => r.question_id);
1344
+ if (failedQuestionIds.length > 0) {
1345
+ const failuresPath = join(runDir, 'failures.json');
1346
+ const failuresData = {
1347
+ questionIds: failedQuestionIds,
1348
+ runId: runDir.split('/').pop(),
1349
+ config: options.memoryConfig,
1350
+ dataset: options.dataset,
1351
+ timestamp: new Date().toISOString(),
1352
+ totalFailed: failedQuestionIds.length,
1353
+ totalQuestions: results.length,
1354
+ };
1355
+ await writeFile(failuresPath, JSON.stringify(failuresData, null, 2));
1356
+ console.log(chalk.gray(`Failures saved to: ${failuresPath}`));
1357
+ }
411
1358
 
412
1359
  // Save metrics
413
1360
  const metricsPath = join(runDir, 'metrics.json');
1361
+ const metricsConfigDef = getMemoryConfig(options.memoryConfig);
414
1362
  const metricsData = {
415
1363
  ...metrics,
416
1364
  config: {
417
1365
  dataset: options.dataset,
418
- model: options.model,
1366
+ model: metricsConfigDef.agentModel,
419
1367
  memoryConfig: options.memoryConfig,
420
1368
  subset: options.subset,
1369
+ // Store OM config for reproducibility
1370
+ ...(metricsConfigDef.usesObservationalMemory && {
1371
+ observationalMemoryConfig: {
1372
+ scope: 'resource',
1373
+ messageTokens: OBSERVATIONAL_MEMORY_DEFAULTS.observation.messageTokens,
1374
+ observationTokens: OBSERVATIONAL_MEMORY_DEFAULTS.reflection.observationTokens,
1375
+ recognizePatterns: false,
1376
+ },
1377
+ }),
421
1378
  },
422
1379
  timestamp: new Date().toISOString(),
423
1380
  };
424
1381
  await writeFile(metricsPath, JSON.stringify(metricsData, null, 2));
425
1382
 
426
- console.log(chalk.gray(`\nResults saved to: ${runDir}`));
1383
+ console.log(
1384
+ chalk.gray(`
1385
+ Results saved to: ${runDir}`),
1386
+ );
427
1387
  }
428
1388
 
429
1389
  private calculateMetrics(results: EvaluationResult[]): BenchmarkMetrics {
@@ -435,15 +1395,33 @@ Be specific rather than generic when the user has expressed clear preferences in
435
1395
  correct_answers: 0,
436
1396
  abstention_correct: 0,
437
1397
  abstention_total: 0,
1398
+ // "Fixed" metrics - uses improved_is_correct where available, otherwise is_correct
1399
+ improved_accuracy: undefined,
1400
+ improved_correct: 0,
1401
+ improved_total: 0,
1402
+ fixed_accuracy_by_type: {} as Record<QuestionType, { correct: number; total: number; accuracy: number }>,
1403
+ fixed_overall_accuracy: undefined,
438
1404
  };
439
1405
 
1406
+ // Check if any results have improved questions
1407
+ const hasAnyImprovedQuestions = results.some(r => r.improved_question !== undefined);
1408
+
440
1409
  // Calculate overall metrics
441
1410
  for (const result of results) {
1411
+ // Vanilla metrics (original question only)
442
1412
  if (result.is_correct) {
443
1413
  metrics.correct_answers++;
444
1414
  }
445
1415
 
446
- // Track by question type
1416
+ // Track how many questions have improved versions
1417
+ if (result.improved_question !== undefined) {
1418
+ metrics.improved_total = (metrics.improved_total || 0) + 1;
1419
+ if (result.improved_is_correct) {
1420
+ metrics.improved_correct = (metrics.improved_correct || 0) + 1;
1421
+ }
1422
+ }
1423
+
1424
+ // Track by question type (vanilla)
447
1425
  if (result.question_type) {
448
1426
  const type = result.question_type;
449
1427
  if (!metrics.accuracy_by_type[type]) {
@@ -453,6 +1431,21 @@ Be specific rather than generic when the user has expressed clear preferences in
453
1431
  if (result.is_correct) {
454
1432
  metrics.accuracy_by_type[type].correct++;
455
1433
  }
1434
+
1435
+ // Track "fixed" metrics by type (use improved result if available, otherwise vanilla)
1436
+ if (hasAnyImprovedQuestions) {
1437
+ if (!metrics.fixed_accuracy_by_type![type]) {
1438
+ metrics.fixed_accuracy_by_type![type] = { correct: 0, total: 0, accuracy: 0 };
1439
+ }
1440
+ metrics.fixed_accuracy_by_type![type].total++;
1441
+
1442
+ // Fixed score: correct if EITHER original OR improved passes
1443
+ // This ensures we don't penalize when improved question regresses
1444
+ const isCorrectFixed = result.is_correct || (result.improved_is_correct ?? false);
1445
+ if (isCorrectFixed) {
1446
+ metrics.fixed_accuracy_by_type![type].correct++;
1447
+ }
1448
+ }
456
1449
  }
457
1450
 
458
1451
  // Track abstention separately
@@ -462,9 +1455,27 @@ Be specific rather than generic when the user has expressed clear preferences in
462
1455
  metrics.abstention_correct = (metrics.abstention_correct || 0) + 1;
463
1456
  }
464
1457
  }
1458
+
1459
+ // Aggregate token usage
1460
+ if (result.usage) {
1461
+ if (!metrics.total_usage) {
1462
+ metrics.total_usage = { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
1463
+ }
1464
+ metrics.total_usage.inputTokens += result.usage.inputTokens;
1465
+ metrics.total_usage.outputTokens += result.usage.outputTokens;
1466
+ metrics.total_usage.totalTokens += result.usage.totalTokens;
1467
+ }
1468
+ if (result.improved_usage) {
1469
+ if (!metrics.improved_total_usage) {
1470
+ metrics.improved_total_usage = { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
1471
+ }
1472
+ metrics.improved_total_usage.inputTokens += result.improved_usage.inputTokens;
1473
+ metrics.improved_total_usage.outputTokens += result.improved_usage.outputTokens;
1474
+ metrics.improved_total_usage.totalTokens += result.improved_usage.totalTokens;
1475
+ }
465
1476
  }
466
1477
 
467
- // Calculate per-type accuracies first
1478
+ // Calculate per-type accuracies (vanilla)
468
1479
  for (const type in metrics.accuracy_by_type) {
469
1480
  const typeMetrics = metrics.accuracy_by_type[type as QuestionType];
470
1481
  if (typeMetrics) {
@@ -472,40 +1483,54 @@ Be specific rather than generic when the user has expressed clear preferences in
472
1483
  }
473
1484
  }
474
1485
 
1486
+ // Calculate per-type accuracies (fixed)
1487
+ if (hasAnyImprovedQuestions && metrics.fixed_accuracy_by_type) {
1488
+ for (const type in metrics.fixed_accuracy_by_type) {
1489
+ const typeMetrics = metrics.fixed_accuracy_by_type[type as QuestionType];
1490
+ if (typeMetrics) {
1491
+ typeMetrics.accuracy = typeMetrics.total > 0 ? typeMetrics.correct / typeMetrics.total : 0;
1492
+ }
1493
+ }
1494
+ }
1495
+
475
1496
  if (metrics.abstention_total && metrics.abstention_total > 0) {
476
1497
  metrics.abstention_accuracy = (metrics.abstention_correct || 0) / metrics.abstention_total;
477
1498
  }
478
1499
 
479
- // Calculate overall accuracy as average of all question type accuracies (excluding abstention)
1500
+ // Calculate overall accuracy as average of all question type accuracies (vanilla)
480
1501
  const allTypeAccuracies = Object.values(metrics.accuracy_by_type).map(t => t.accuracy);
481
-
482
- // Debug: Log the exact values being averaged
483
- console.log('\nDebug - Question type accuracies being averaged:');
484
- Object.entries(metrics.accuracy_by_type).forEach(([type, data]) => {
485
- console.log(` ${type}: ${data.accuracy} (${data.correct}/${data.total})`);
486
- });
487
- console.log(` Sum: ${allTypeAccuracies.reduce((sum, acc) => sum + acc, 0)}`);
488
- console.log(` Count: ${allTypeAccuracies.length}`);
489
-
490
1502
  metrics.overall_accuracy =
491
1503
  allTypeAccuracies.length > 0
492
1504
  ? allTypeAccuracies.reduce((sum, acc) => sum + acc, 0) / allTypeAccuracies.length
493
1505
  : 0;
494
1506
 
495
- console.log(` Calculated overall: ${metrics.overall_accuracy}`);
496
- console.log(` As percentage: ${(metrics.overall_accuracy * 100).toFixed(10)}%\n`);
1507
+ // Calculate fixed overall accuracy
1508
+ if (hasAnyImprovedQuestions && metrics.fixed_accuracy_by_type) {
1509
+ const fixedTypeAccuracies = Object.values(metrics.fixed_accuracy_by_type).map(t => t.accuracy);
1510
+ metrics.fixed_overall_accuracy =
1511
+ fixedTypeAccuracies.length > 0
1512
+ ? fixedTypeAccuracies.reduce((sum, acc) => sum + acc, 0) / fixedTypeAccuracies.length
1513
+ : 0;
1514
+ }
497
1515
 
498
1516
  return metrics;
499
1517
  }
500
1518
 
501
- private displayMetrics(metrics: BenchmarkMetrics, options?: RunOptions): void {
502
- console.log(chalk.bold('\n📊 Benchmark Results\n'));
1519
+ private displayMetrics(metrics: BenchmarkMetrics, options?: RunOptions, configDef?: MemoryConfigDefinition): void {
1520
+ console.log(
1521
+ chalk.bold(`
1522
+ 📊 Benchmark Results
1523
+ `),
1524
+ );
503
1525
 
504
1526
  // Display configuration if provided
505
1527
  if (options) {
506
- console.log(chalk.bold('Configuration:\n'));
1528
+ console.log(
1529
+ chalk.bold(`Configuration:
1530
+ `),
1531
+ );
507
1532
  console.log(chalk.gray('Dataset:'), chalk.cyan(options.dataset));
508
- console.log(chalk.gray('Model:'), chalk.cyan(options.model));
1533
+ console.log(chalk.gray('Model:'), chalk.cyan(configDef?.agentModel ?? 'gpt-4o'));
509
1534
  console.log(chalk.gray('Memory Config:'), chalk.cyan(options.memoryConfig));
510
1535
  if (options.subset) {
511
1536
  console.log(chalk.gray('Subset:'), chalk.cyan(`${options.subset} questions`));
@@ -517,37 +1542,131 @@ Be specific rather than generic when the user has expressed clear preferences in
517
1542
  console.log();
518
1543
  }
519
1544
 
1545
+ // Check if we have fixed metrics to display
1546
+ const hasFixedMetrics = metrics.fixed_accuracy_by_type && Object.keys(metrics.fixed_accuracy_by_type).length > 0;
1547
+
520
1548
  // Question type breakdown
521
1549
  console.log(chalk.bold('Accuracy by Question Type:'));
522
1550
 
523
1551
  // Sort question types alphabetically
524
1552
  const sortedTypes = Object.entries(metrics.accuracy_by_type).sort(([a], [b]) => a.localeCompare(b));
525
1553
 
526
- // Display regular question types
1554
+ // Helper to create progress bar
1555
+ const createBar = (accuracy: number, length: number = 20) => {
1556
+ const filledLength = Math.round(accuracy * length);
1557
+ return '█'.repeat(filledLength) + '░'.repeat(length - filledLength);
1558
+ };
1559
+
1560
+ // Helper to get color based on accuracy
1561
+ const getColor = (accuracy: number) => (accuracy >= 0.8 ? 'green' : accuracy >= 0.6 ? 'yellow' : 'red');
1562
+
1563
+ // Display regular question types (vanilla)
527
1564
  for (const [type, typeMetrics] of sortedTypes) {
528
1565
  const { correct, total, accuracy } = typeMetrics;
529
- const typeColor = accuracy >= 0.8 ? 'green' : accuracy >= 0.6 ? 'yellow' : 'red';
530
-
531
- // Create a simple progress bar
532
- const barLength = 20;
533
- const filledLength = Math.round(accuracy * barLength);
534
- const bar = '█'.repeat(filledLength) + '░'.repeat(barLength - filledLength);
1566
+ const typeColor = getColor(accuracy);
535
1567
 
536
1568
  console.log(
537
1569
  chalk.gray(` ${type.padEnd(25)}:`),
538
1570
  chalk[typeColor](`${(accuracy * 100).toFixed(1).padStart(5)}%`),
539
- chalk.gray(`[${bar}]`),
1571
+ chalk.gray(`[${createBar(accuracy)}]`),
540
1572
  chalk.gray(`(${correct}/${total})`),
541
1573
  );
1574
+
1575
+ // If we have fixed metrics, show the fixed version right after
1576
+ if (hasFixedMetrics && metrics.fixed_accuracy_by_type![type as QuestionType]) {
1577
+ const fixedTypeMetrics = metrics.fixed_accuracy_by_type![type as QuestionType];
1578
+ const fixedColor = getColor(fixedTypeMetrics.accuracy);
1579
+ console.log(
1580
+ chalk.gray(` ${(type + ' (fixed)').padEnd(25)}:`),
1581
+ chalk[fixedColor](`${(fixedTypeMetrics.accuracy * 100).toFixed(1).padStart(5)}%`),
1582
+ chalk.gray(`[${createBar(fixedTypeMetrics.accuracy)}]`),
1583
+ chalk.gray(`(${fixedTypeMetrics.correct}/${fixedTypeMetrics.total})`),
1584
+ );
1585
+ }
542
1586
  }
543
1587
 
544
1588
  console.log();
545
- const accuracyColor =
546
- metrics.overall_accuracy >= 0.8 ? 'green' : metrics.overall_accuracy >= 0.6 ? 'yellow' : 'red';
1589
+
1590
+ // Overall accuracy (vanilla)
1591
+ const accuracyColor = getColor(metrics.overall_accuracy);
547
1592
  console.log(
548
- chalk.bold('Overall Accuracy:'),
1593
+ chalk.bold('Overall Accuracy: '),
549
1594
  chalk[accuracyColor](`${(metrics.overall_accuracy * 100).toFixed(2)}%`),
550
1595
  chalk.gray(`(average of ${Object.keys(metrics.accuracy_by_type).length} question types)`),
551
1596
  );
1597
+
1598
+ // Overall accuracy (fixed) - shown if any improved questions exist
1599
+ if (hasFixedMetrics && metrics.fixed_overall_accuracy !== undefined) {
1600
+ const fixedAccuracyColor = getColor(metrics.fixed_overall_accuracy);
1601
+ console.log(
1602
+ chalk.bold('Overall Accuracy (fixed):'),
1603
+ chalk[fixedAccuracyColor](`${(metrics.fixed_overall_accuracy * 100).toFixed(2)}%`),
1604
+ chalk.gray(`(${metrics.improved_total} questions clarified)`),
1605
+ );
1606
+ }
1607
+
1608
+ // Token usage summary
1609
+ if (metrics.total_usage) {
1610
+ console.log();
1611
+ console.log(chalk.bold('Token Usage:'));
1612
+ const formatTokens = (n: number) => n.toLocaleString();
1613
+ console.log(
1614
+ chalk.gray(' Input tokens: '),
1615
+ chalk.cyan(formatTokens(metrics.total_usage.inputTokens)),
1616
+ chalk.gray(
1617
+ `(avg ${formatTokens(Math.round(metrics.total_usage.inputTokens / metrics.total_questions))}/question)`,
1618
+ ),
1619
+ );
1620
+ console.log(chalk.gray(' Output tokens:'), chalk.cyan(formatTokens(metrics.total_usage.outputTokens)));
1621
+ console.log(chalk.gray(' Total tokens: '), chalk.cyan(formatTokens(metrics.total_usage.totalTokens)));
1622
+
1623
+ if (metrics.improved_total_usage) {
1624
+ console.log(
1625
+ chalk.gray(' Improved input tokens: '),
1626
+ chalk.cyan(formatTokens(metrics.improved_total_usage.inputTokens)),
1627
+ );
1628
+ }
1629
+ }
1630
+ }
1631
+
1632
+ private displayUninvestigatedFailures(results: EvaluationResult[]): void {
1633
+ // Find failures that have no improvement info (not yet investigated)
1634
+ const uninvestigatedFailures = results.filter(r => r.is_correct === false && !r.has_improvement_info);
1635
+
1636
+ if (uninvestigatedFailures.length === 0) {
1637
+ return;
1638
+ }
1639
+
1640
+ console.log(
1641
+ chalk.yellow(`
1642
+ 🔍 Failures for Investigation (${uninvestigatedFailures.length})
1643
+ `),
1644
+ );
1645
+ console.log(
1646
+ chalk.gray('These questions failed and have no improved_question, improved_answer, or improvement_note:\n'),
1647
+ );
1648
+
1649
+ // Group by question type for easier review
1650
+ const byType = new Map<string, EvaluationResult[]>();
1651
+ for (const result of uninvestigatedFailures) {
1652
+ const type = result.question_type || 'unknown';
1653
+ if (!byType.has(type)) {
1654
+ byType.set(type, []);
1655
+ }
1656
+ byType.get(type)!.push(result);
1657
+ }
1658
+
1659
+ // Display grouped by type
1660
+ for (const [type, failures] of Array.from(byType.entries()).sort(([a], [b]) => a.localeCompare(b))) {
1661
+ console.log(chalk.cyan(` ${type}:`));
1662
+ for (const result of failures) {
1663
+ console.log(chalk.gray(` - ${result.question_id}`));
1664
+ console.log(chalk.gray(` Q: "${result.question}"`));
1665
+ console.log(chalk.gray(` A: "${result.hypothesis}"`));
1666
+ console.log(chalk.yellow(` Expected: "${result.expected_answer}"`));
1667
+ }
1668
+ }
1669
+
1670
+ console.log();
552
1671
  }
553
1672
  }