@mastra/longmemeval 0.0.0-error-handler-fix-20251020202607 → 0.0.0-execa-dynamic-import-20260304221256

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/CHANGELOG.md +847 -6
  2. package/LICENSE.md +15 -0
  3. package/docs/INVESTIGATION_WORKFLOW.md +314 -0
  4. package/package.json +34 -26
  5. package/scripts/download.ts +22 -1
  6. package/scripts/generate-wm-templates.ts +2 -1
  7. package/src/cli.ts +967 -155
  8. package/src/commands/__tests__/prepare.test.ts +11 -6
  9. package/src/commands/__tests__/run.test.ts +10 -8
  10. package/src/commands/clean.ts +147 -0
  11. package/src/commands/deterministic-ids.ts +126 -0
  12. package/src/commands/find-prohibited.ts +299 -0
  13. package/src/commands/investigate.ts +2437 -0
  14. package/src/commands/list-partial.ts +137 -0
  15. package/src/commands/obscure-thread-ids.ts +237 -0
  16. package/src/commands/partial-results.ts +198 -0
  17. package/src/commands/precompute-embeddings.ts +302 -0
  18. package/src/commands/prepare.ts +732 -105
  19. package/src/commands/reconcile.ts +383 -0
  20. package/src/commands/run.ts +1240 -121
  21. package/src/commands/sessions.ts +126 -0
  22. package/src/commands/sync.ts +152 -0
  23. package/src/commands/test-cleaned.ts +53 -0
  24. package/src/commands/test-incremental.ts +120 -0
  25. package/src/commands/test-messages.ts +95 -0
  26. package/src/commands/test-prohibited.ts +244 -0
  27. package/src/commands/test-thread.ts +136 -0
  28. package/src/commands/tokens.ts +625 -0
  29. package/src/config.ts +1253 -72
  30. package/src/data/types.ts +48 -6
  31. package/src/embeddings/cached-openai-embedding-model.ts +0 -1
  32. package/src/evaluation/longmemeval-metric.ts +186 -103
  33. package/src/processors/content-sanitizer.ts +153 -0
  34. package/src/processors/date-injector.ts +111 -0
  35. package/src/processors/index.ts +3 -0
  36. package/src/processors/observation-semantic-filter.ts +1142 -0
  37. package/src/retry-model.ts +5 -3
  38. package/src/storage/__tests__/benchmark-store.test.ts +18 -35
  39. package/src/storage/benchmark-store.ts +140 -153
  40. package/src/storage/benchmark-vector.ts +1 -1
  41. package/src/storage/index.ts +1 -0
  42. package/src/storage/persistable-inmemory.ts +133 -0
  43. package/src/test-utils/mock-model.ts +35 -0
@@ -0,0 +1,2437 @@
1
+ import { existsSync } from 'fs';
2
+ import { readFile, writeFile, mkdir, readdir } from 'fs/promises';
3
+ import { join, dirname } from 'path';
4
+ import { DatasetLoader } from '../data/loader';
5
+ import { PersistableInMemoryMemory } from '../storage';
6
+ import type { LongMemEvalQuestion, FailureCategory } from '../data/types';
7
+
8
+ // Re-export FailureCategory for backwards compatibility
9
+ export type { FailureCategory } from '../data/types';
10
+
11
+ export interface QuestionProgress {
12
+ status: 'pending' | 'investigated' | 'fix-implemented' | 'synced';
13
+ category?: FailureCategory;
14
+ investigatedAt?: string;
15
+ }
16
+
17
+ export interface InvestigationProgress {
18
+ runId: string;
19
+ config: string;
20
+ dataset: string;
21
+ createdAt: string;
22
+ totalFailed: number;
23
+ investigated: number;
24
+ questions: Record<string, QuestionProgress>;
25
+ }
26
+
27
+ export interface InvestigateOptions {
28
+ runId?: string;
29
+ list?: boolean;
30
+ config?: string;
31
+ status?: boolean;
32
+ next?: boolean;
33
+ done?: string;
34
+ sync?: boolean;
35
+ fixed?: string; // Mark a question as fix-implemented
36
+ outputDir?: string;
37
+ resultsDir?: string;
38
+ preparedDataDir?: string;
39
+ datasetDir?: string;
40
+ editor?: string;
41
+ // Investigation utilities
42
+ search?: string; // Search observations for keywords
43
+ trace?: string; // Trace information flow for a keyword
44
+ questionId?: string; // Question ID for search/trace/date/session
45
+ inspect?: string; // Inspect a specific question's data
46
+ date?: string; // View observations around a specific date (e.g., "2023/05/29" or "May 29")
47
+ context?: number; // Number of days of context around the date (default: 1)
48
+ session?: number; // View a specific session from original dataset
49
+ listSessions?: boolean; // List all sessions with dates for a question
50
+ // Data freshness detection
51
+ checkStale?: boolean; // Check if prepared data is stale (pre-cursor-fix)
52
+ staleOnly?: boolean; // Only list stale questions from failures
53
+ // Original dataset search
54
+ searchOriginal?: string; // Search original dataset for a keyword
55
+ // Improve question/answer in dataset
56
+ improve?: string; // Question ID to add improvements for
57
+ improveQuestion?: string; // Improved question text
58
+ improveAnswer?: string; // Improved answer text
59
+ improveNote?: string; // Improvement note
60
+ category?: FailureCategory; // Failure category for this question
61
+ clearImproved?: string | boolean; // Clear specific field(s): 'all', 'question', 'answer', 'note', 'category' or true for all
62
+ // Check for duplicate observations
63
+ checkDuplicates?: boolean; // Check for duplicate thread blocks in observations
64
+ // Baseline check - comprehensive data quality check
65
+ baseline?: boolean; // Run all data quality checks for a question
66
+ // Prepare stale questions
67
+ prepareStale?: boolean; // Find and prepare all stale pending questions (deprecated, use --print-prepare-command)
68
+ dryRun?: boolean; // Just show what would be prepared, don't run
69
+ printPrepareCommand?: boolean; // Print a prepare command for stale questions
70
+ }
71
+
72
+ interface FailuresFile {
73
+ questionIds: string[];
74
+ runId: string;
75
+ config: string;
76
+ dataset: string;
77
+ timestamp: string;
78
+ totalFailed: number;
79
+ totalQuestions: number;
80
+ }
81
+
82
+ interface EvaluationResult {
83
+ question_id: string;
84
+ question: string;
85
+ expected_answer: string;
86
+ hypothesis: string; // The agent's answer
87
+ is_correct: boolean;
88
+ question_type: string;
89
+ improved_question?: string;
90
+ improved_answer?: string;
91
+ improved_is_correct?: boolean;
92
+ has_improvement_info?: boolean;
93
+ improved_regression?: boolean;
94
+ }
95
+
96
+ // ============================================================================
97
+ // Analysis Template
98
+ // ============================================================================
99
+
100
+ function generateAnalysisTemplate(
101
+ questionId: string,
102
+ question: string,
103
+ expectedAnswer: string,
104
+ actualAnswer: string,
105
+ questionType: string,
106
+ improvedQuestion?: string,
107
+ improvedAnswer?: string,
108
+ ): string {
109
+ return `# Investigation: ${questionId}
110
+
111
+ ## Question Type
112
+ ${questionType}
113
+
114
+ ## Question
115
+ ${question}
116
+
117
+ ## Expected Answer
118
+ ${expectedAnswer}
119
+
120
+ ## Agent Answer
121
+ ${actualAnswer}
122
+
123
+ ${improvedQuestion ? `## Improved Question (existing)\n${improvedQuestion}\n` : ''}
124
+ ${improvedAnswer ? `## Improved Answer (existing)\n${improvedAnswer}\n` : ''}
125
+
126
+ ---
127
+
128
+ ## Failure Category
129
+ - [ ] Observer missed critical information
130
+ - [ ] Reflector lost/merged information incorrectly
131
+ - [ ] Agent reasoning error (had info, wrong conclusion)
132
+ - [ ] Ambiguous/poorly-worded question
133
+ - [ ] Dataset inconsistency/error
134
+ - [ ] RAG retrieval miss (if applicable)
135
+ - [ ] Other: ___
136
+
137
+ ## Root Cause Analysis
138
+ <!-- What specifically went wrong? -->
139
+
140
+
141
+ ## Evidence
142
+ <!-- Quote relevant parts of om.md, original data, etc. -->
143
+
144
+
145
+ ---
146
+
147
+ ## Potential Improvements
148
+
149
+ ### Observer/Reflector Changes
150
+ - **Likelihood**: Low / Medium / High
151
+ - **Reasoning**:
152
+ - **Suggested prompt change**:
153
+
154
+ ### Fixed Question/Answer
155
+ - **Likelihood**: Low / Medium / High
156
+ - **improved_question**:
157
+ - **improved_answer**:
158
+ - **improvement_note**:
159
+
160
+ ### Other Improvements
161
+ <!-- Prompt changes, architectural changes, etc. -->
162
+
163
+
164
+ ---
165
+
166
+ ## Status
167
+ - [ ] Investigated
168
+ - [ ] Fix implemented
169
+ - [ ] Synced to longmemeval_s.json
170
+ `;
171
+ }
172
+
173
+ // ============================================================================
174
+ // Investigation Command
175
+ // ============================================================================
176
+
177
+ export class InvestigateCommand {
178
+ private investigationsDir: string;
179
+ private resultsDir: string;
180
+ private preparedDataDir: string;
181
+ private datasetDir: string;
182
+ private editor: string;
183
+
184
+ constructor(options: InvestigateOptions = {}) {
185
+ this.investigationsDir = options.outputDir || './investigations';
186
+ this.resultsDir = options.resultsDir || './results';
187
+ this.preparedDataDir = options.preparedDataDir || './prepared-data';
188
+ this.datasetDir = options.datasetDir || './data';
189
+ this.editor = options.editor || process.env.EDITOR || 'code';
190
+ }
191
+
192
+ async run(options: InvestigateOptions): Promise<void> {
193
+ // Handle different modes
194
+ if (options.list) {
195
+ await this.listFailures(options.config);
196
+ return;
197
+ }
198
+
199
+ if (options.status) {
200
+ await this.showStatus();
201
+ return;
202
+ }
203
+
204
+ if (options.next) {
205
+ await this.openNext();
206
+ return;
207
+ }
208
+
209
+ if (options.done) {
210
+ await this.markDone(options.done);
211
+ return;
212
+ }
213
+
214
+ if (options.fixed) {
215
+ await this.markFixed(options.fixed);
216
+ return;
217
+ }
218
+
219
+ if (options.sync) {
220
+ await this.syncToDataset();
221
+ return;
222
+ }
223
+
224
+ // Investigation utilities
225
+ if (options.inspect) {
226
+ await this.inspectQuestion(options.inspect);
227
+ return;
228
+ }
229
+
230
+ if (options.search && options.questionId) {
231
+ await this.searchObservations(options.questionId, options.search);
232
+ return;
233
+ }
234
+
235
+ if (options.trace && options.questionId) {
236
+ await this.traceInformation(options.questionId, options.trace);
237
+ return;
238
+ }
239
+
240
+ // Date-based observation viewer
241
+ if (options.date && options.questionId) {
242
+ await this.viewObservationsAroundDate(options.questionId, options.date, options.context ?? 1);
243
+ return;
244
+ }
245
+
246
+ // Session viewer
247
+ if (options.listSessions && options.questionId) {
248
+ await this.listSessions(options.questionId);
249
+ return;
250
+ }
251
+
252
+ if (options.session !== undefined && options.questionId) {
253
+ await this.viewSession(options.questionId, options.session);
254
+ return;
255
+ }
256
+
257
+ // Data freshness detection
258
+ if (options.checkStale) {
259
+ await this.checkStaleData(options.questionId, options.staleOnly);
260
+ return;
261
+ }
262
+
263
+ // Original dataset search
264
+ if (options.searchOriginal && options.questionId) {
265
+ await this.searchOriginalDataset(options.questionId, options.searchOriginal);
266
+ return;
267
+ }
268
+
269
+ // Improve question/answer in dataset
270
+ if (options.improve) {
271
+ await this.addImprovement(
272
+ options.improve,
273
+ options.improveQuestion,
274
+ options.improveAnswer,
275
+ options.improveNote,
276
+ options.category,
277
+ options.clearImproved,
278
+ );
279
+ return;
280
+ }
281
+
282
+ // Check for duplicate observations
283
+ if (options.checkDuplicates) {
284
+ await this.checkDuplicateObservations(options.questionId);
285
+ return;
286
+ }
287
+
288
+ // Baseline check - comprehensive data quality check
289
+ if (options.baseline && options.questionId) {
290
+ await this.runBaselineCheck(options.questionId);
291
+ return;
292
+ }
293
+
294
+ // Prepare stale questions (deprecated)
295
+ if (options.prepareStale) {
296
+ await this.prepareStaleQuestions(options.dryRun);
297
+ return;
298
+ }
299
+
300
+ // Print prepare command for stale questions
301
+ if (options.printPrepareCommand) {
302
+ await this.printPrepareCommand();
303
+ return;
304
+ }
305
+
306
+ // Default: setup investigation from run
307
+ if (options.runId) {
308
+ await this.setupInvestigation(options.runId);
309
+ return;
310
+ }
311
+
312
+ // No args: show help
313
+ this.showHelp();
314
+ }
315
+
316
+ // --------------------------------------------------------------------------
317
+ // List Failures
318
+ // --------------------------------------------------------------------------
319
+
320
+ private async listFailures(configFilter?: string): Promise<void> {
321
+ console.log('\nšŸ” Scanning for benchmark runs with failures...\n');
322
+
323
+ interface RunInfo {
324
+ runId: string;
325
+ config: string;
326
+ dataset: string;
327
+ timestamp: string;
328
+ totalFailed: number;
329
+ totalQuestions: number;
330
+ failuresPath: string;
331
+ }
332
+
333
+ const runs: RunInfo[] = [];
334
+
335
+ // Scan results directory for all configs
336
+ if (!existsSync(this.resultsDir)) {
337
+ console.log('No results directory found.');
338
+ return;
339
+ }
340
+
341
+ const configs = await readdir(this.resultsDir);
342
+
343
+ for (const config of configs) {
344
+ // Skip if filtering by config and doesn't match
345
+ if (configFilter && !config.includes(configFilter)) {
346
+ continue;
347
+ }
348
+
349
+ const configDir = join(this.resultsDir, config);
350
+ const stat = await import('fs/promises').then(fs => fs.stat(configDir));
351
+ if (!stat.isDirectory()) continue;
352
+
353
+ const runDirs = await readdir(configDir);
354
+
355
+ for (const runDir of runDirs) {
356
+ if (!runDir.startsWith('run_')) continue;
357
+
358
+ const failuresPath = join(configDir, runDir, 'failures.json');
359
+ if (!existsSync(failuresPath)) continue;
360
+
361
+ try {
362
+ const failures = JSON.parse(await readFile(failuresPath, 'utf-8')) as FailuresFile;
363
+ runs.push({
364
+ runId: failures.runId,
365
+ config: failures.config,
366
+ dataset: failures.dataset,
367
+ timestamp: failures.timestamp,
368
+ totalFailed: failures.totalFailed,
369
+ totalQuestions: failures.totalQuestions,
370
+ failuresPath,
371
+ });
372
+ } catch {
373
+ // Skip invalid files
374
+ }
375
+ }
376
+ }
377
+
378
+ if (runs.length === 0) {
379
+ console.log('No runs with failures found.');
380
+ return;
381
+ }
382
+
383
+ // Sort by timestamp (newest first)
384
+ runs.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
385
+
386
+ // Group by config
387
+ const byConfig = new Map<string, RunInfo[]>();
388
+ for (const run of runs) {
389
+ const key = `${run.dataset}/${run.config}`;
390
+ if (!byConfig.has(key)) {
391
+ byConfig.set(key, []);
392
+ }
393
+ byConfig.get(key)!.push(run);
394
+ }
395
+
396
+ // Display
397
+ console.log(`Found ${runs.length} runs with failures across ${byConfig.size} configs:\n`);
398
+
399
+ for (const [configKey, configRuns] of byConfig) {
400
+ console.log(`šŸ“ ${configKey}`);
401
+
402
+ // Show latest run prominently
403
+ const latest = configRuns[0];
404
+ const failRate = ((latest.totalFailed / latest.totalQuestions) * 100).toFixed(1);
405
+ console.log(` Latest: ${latest.runId}`);
406
+ console.log(` Failed: ${latest.totalFailed}/${latest.totalQuestions} (${failRate}%)`);
407
+ console.log(` Date: ${new Date(latest.timestamp).toLocaleString()}`);
408
+
409
+ if (configRuns.length > 1) {
410
+ console.log(` (${configRuns.length - 1} older runs)`);
411
+ }
412
+ console.log();
413
+ }
414
+
415
+ // Show usage hint
416
+ console.log('To investigate a run:');
417
+ console.log(` pnpm investigate <run-id>`);
418
+ console.log(` pnpm investigate ${runs[0].runId}`);
419
+ }
420
+
421
+ // --------------------------------------------------------------------------
422
+ // Setup Investigation
423
+ // --------------------------------------------------------------------------
424
+
425
+ private async setupInvestigation(runIdOrPath: string): Promise<void> {
426
+ console.log(`\nšŸ” Setting up investigation for: ${runIdOrPath}\n`);
427
+
428
+ // Find the failures.json file
429
+ const { failuresPath, failures } = await this.findFailures(runIdOrPath);
430
+ console.log(`šŸ“ Found failures file: ${failuresPath}`);
431
+ console.log(` Config: ${failures.config}`);
432
+ console.log(` Dataset: ${failures.dataset}`);
433
+ console.log(` Failed: ${failures.totalFailed}/${failures.totalQuestions}`);
434
+
435
+ // Create investigation directory
436
+ const investigationDir = join(this.investigationsDir, failures.runId);
437
+ await mkdir(investigationDir, { recursive: true });
438
+
439
+ // Load existing progress or create new
440
+ const progressPath = join(investigationDir, 'progress.json');
441
+ let progress: InvestigationProgress;
442
+
443
+ if (existsSync(progressPath)) {
444
+ progress = JSON.parse(await readFile(progressPath, 'utf-8'));
445
+ console.log(`\nšŸ“Š Resuming existing investigation (${progress.investigated}/${progress.totalFailed} done)`);
446
+ } else {
447
+ progress = {
448
+ runId: failures.runId,
449
+ config: failures.config,
450
+ dataset: failures.dataset,
451
+ createdAt: new Date().toISOString(),
452
+ totalFailed: failures.totalFailed,
453
+ investigated: 0,
454
+ questions: {},
455
+ };
456
+
457
+ for (const qid of failures.questionIds) {
458
+ progress.questions[qid] = { status: 'pending' };
459
+ }
460
+ }
461
+
462
+ // Load results.jsonl to get actual answers
463
+ const resultsDir = dirname(failuresPath);
464
+ const resultsPath = join(resultsDir, 'results.jsonl');
465
+ const resultsMap = new Map<string, EvaluationResult>();
466
+
467
+ if (existsSync(resultsPath)) {
468
+ const resultsContent = await readFile(resultsPath, 'utf-8');
469
+ for (const line of resultsContent.split('\n').filter(l => l.trim())) {
470
+ try {
471
+ const result = JSON.parse(line) as EvaluationResult;
472
+ resultsMap.set(result.question_id, result);
473
+ } catch {
474
+ // Skip invalid lines
475
+ }
476
+ }
477
+ }
478
+
479
+ // Load dataset for original questions
480
+ const loader = new DatasetLoader(this.datasetDir);
481
+ const dataset = await loader.loadDataset(failures.dataset as 'longmemeval_s' | 'longmemeval_m');
482
+ const questionMap = new Map<string, LongMemEvalQuestion>();
483
+ for (const q of dataset) {
484
+ questionMap.set(q.question_id, q);
485
+ }
486
+
487
+ // Setup each failed question
488
+ let created = 0;
489
+ let skipped = 0;
490
+
491
+ for (const questionId of failures.questionIds) {
492
+ const questionDir = join(investigationDir, questionId);
493
+ const dataDir = join(questionDir, 'data');
494
+ const analysisPath = join(questionDir, 'analysis.md');
495
+
496
+ // Skip if already exists
497
+ if (existsSync(analysisPath)) {
498
+ skipped++;
499
+ continue;
500
+ }
501
+
502
+ await mkdir(dataDir, { recursive: true });
503
+
504
+ // Get question data
505
+ const question = questionMap.get(questionId);
506
+ const result = resultsMap.get(questionId);
507
+
508
+ if (!question) {
509
+ console.warn(` āš ļø Question ${questionId} not found in dataset`);
510
+ continue;
511
+ }
512
+
513
+ // Copy original question data
514
+ await writeFile(join(dataDir, 'original.json'), JSON.stringify(question, null, 2));
515
+
516
+ // Copy result if available
517
+ if (result) {
518
+ await writeFile(join(dataDir, 'result.json'), JSON.stringify(result, null, 2));
519
+ }
520
+
521
+ // Copy prepared data files
522
+ const preparedDir = join(this.preparedDataDir, failures.dataset, failures.config, questionId);
523
+
524
+ const filesToCopy = ['om.md', 'om.json', 'meta.json'];
525
+ for (const file of filesToCopy) {
526
+ const srcPath = join(preparedDir, file);
527
+ if (existsSync(srcPath)) {
528
+ const content = await readFile(srcPath, 'utf-8');
529
+ await writeFile(join(dataDir, file), content);
530
+ }
531
+ }
532
+
533
+ // Generate analysis template
534
+ const template = generateAnalysisTemplate(
535
+ questionId,
536
+ question.question,
537
+ question.answer,
538
+ result?.hypothesis || '(not available)',
539
+ question.question_type,
540
+ question.improved_question,
541
+ question.improved_answer,
542
+ );
543
+
544
+ await writeFile(analysisPath, template);
545
+ created++;
546
+ }
547
+
548
+ // Save progress
549
+ await writeFile(progressPath, JSON.stringify(progress, null, 2));
550
+
551
+ console.log(`\nāœ… Investigation setup complete!`);
552
+ console.log(` Created: ${created} new question directories`);
553
+ console.log(` Skipped: ${skipped} existing directories`);
554
+ console.log(` Location: ${investigationDir}`);
555
+ console.log(`\nšŸ“ Next steps:`);
556
+ console.log(` pnpm investigate --status # Check progress`);
557
+ console.log(` pnpm investigate --next # Open next question`);
558
+ console.log(` pnpm investigate --done <question-id> # Mark as investigated`);
559
+ }
560
+
561
+ // --------------------------------------------------------------------------
562
+ // Status
563
+ // --------------------------------------------------------------------------
564
+
565
+ private async showStatus(): Promise<void> {
566
+ const investigations = await this.listInvestigations();
567
+
568
+ if (investigations.length === 0) {
569
+ console.log('\nšŸ“­ No investigations found.');
570
+ console.log(' Run: pnpm investigate <run-id> to start one.\n');
571
+ return;
572
+ }
573
+
574
+ console.log('\nšŸ“Š Investigation Status\n');
575
+
576
+ for (const inv of investigations) {
577
+ const progress = await this.loadProgress(inv);
578
+ if (!progress) continue;
579
+
580
+ const pending = Object.values(progress.questions).filter(q => q.status === 'pending').length;
581
+ const investigated = Object.values(progress.questions).filter(q => q.status === 'investigated').length;
582
+ const fixImplemented = Object.values(progress.questions).filter(q => q.status === 'fix-implemented').length;
583
+ const synced = Object.values(progress.questions).filter(q => q.status === 'synced').length;
584
+
585
+ const pct = (((progress.totalFailed - pending) / progress.totalFailed) * 100).toFixed(1);
586
+
587
+ console.log(`šŸ“ ${inv}`);
588
+ console.log(` Config: ${progress.config}`);
589
+ console.log(` Progress: ${progress.totalFailed - pending}/${progress.totalFailed} (${pct}%)`);
590
+ console.log(` ā”œā”€ Pending: ${pending}`);
591
+ console.log(` ā”œā”€ Investigated: ${investigated}`);
592
+ console.log(` ā”œā”€ Fix Implemented: ${fixImplemented}`);
593
+ console.log(` └─ Synced: ${synced}`);
594
+
595
+ // Show category breakdown if any investigated
596
+ if (investigated + fixImplemented + synced > 0) {
597
+ const categories = new Map<string, number>();
598
+ for (const q of Object.values(progress.questions)) {
599
+ if (q.category) {
600
+ categories.set(q.category, (categories.get(q.category) || 0) + 1);
601
+ }
602
+ }
603
+
604
+ if (categories.size > 0) {
605
+ console.log(` Categories:`);
606
+ for (const [cat, count] of [...categories.entries()].sort((a, b) => b[1] - a[1])) {
607
+ console.log(` ${cat}: ${count}`);
608
+ }
609
+ }
610
+ }
611
+
612
+ console.log('');
613
+ }
614
+ }
615
+
616
+ // --------------------------------------------------------------------------
617
+ // Open Next
618
+ // --------------------------------------------------------------------------
619
+
620
+ private async openNext(): Promise<void> {
621
+ const investigations = await this.listInvestigations();
622
+
623
+ if (investigations.length === 0) {
624
+ console.log('\nšŸ“­ No investigations found.\n');
625
+ return;
626
+ }
627
+
628
+ // Use most recent investigation
629
+ const latestInv = investigations[investigations.length - 1];
630
+ const progress = await this.loadProgress(latestInv);
631
+
632
+ if (!progress) {
633
+ console.log('\nāŒ Could not load progress.\n');
634
+ return;
635
+ }
636
+
637
+ // Find next pending question
638
+ const pendingId = Object.entries(progress.questions).find(([_, q]) => q.status === 'pending')?.[0];
639
+
640
+ if (!pendingId) {
641
+ console.log('\nšŸŽ‰ All questions investigated!\n');
642
+ return;
643
+ }
644
+
645
+ const analysisPath = join(this.investigationsDir, latestInv, pendingId, 'analysis.md');
646
+
647
+ if (!existsSync(analysisPath)) {
648
+ console.log(`\nāŒ Analysis file not found: ${analysisPath}\n`);
649
+ return;
650
+ }
651
+
652
+ console.log(`\nšŸ“ Opening: ${pendingId}`);
653
+ console.log(` File: ${analysisPath}`);
654
+ console.log(` Editor: ${this.editor}\n`);
655
+
656
+ // Open in editor
657
+ const { spawn } = await import('child_process');
658
+ spawn(this.editor, [analysisPath], {
659
+ detached: true,
660
+ stdio: 'ignore',
661
+ }).unref();
662
+
663
+ // Also print some context
664
+ const dataDir = join(this.investigationsDir, latestInv, pendingId, 'data');
665
+ const resultPath = join(dataDir, 'result.json');
666
+
667
+ if (existsSync(resultPath)) {
668
+ const result = JSON.parse(await readFile(resultPath, 'utf-8')) as EvaluationResult;
669
+ const expectedStr = String(result.expected_answer);
670
+ const hypothesisStr = String(result.hypothesis);
671
+ console.log(`šŸ“‹ Quick Context:`);
672
+ console.log(` Type: ${result.question_type}`);
673
+ console.log(` Q: ${result.question.substring(0, 100)}...`);
674
+ console.log(` Expected: ${expectedStr.substring(0, 100)}${expectedStr.length > 100 ? '...' : ''}`);
675
+ console.log(` Got: ${hypothesisStr.substring(0, 100)}${hypothesisStr.length > 100 ? '...' : ''}`);
676
+ console.log('');
677
+ }
678
+
679
+ console.log(`When done, run: pnpm investigate --done ${pendingId}\n`);
680
+ }
681
+
682
+ // --------------------------------------------------------------------------
683
+ // Mark Done
684
+ // --------------------------------------------------------------------------
685
+
686
+ private async markDone(questionId: string): Promise<void> {
687
+ const investigations = await this.listInvestigations();
688
+
689
+ if (investigations.length === 0) {
690
+ console.log('\nšŸ“­ No investigations found.\n');
691
+ return;
692
+ }
693
+
694
+ // Find investigation containing this question
695
+ let foundInv: string | null = null;
696
+ let progress: InvestigationProgress | null = null;
697
+
698
+ for (const inv of investigations) {
699
+ const p = await this.loadProgress(inv);
700
+ if (p && questionId in p.questions) {
701
+ foundInv = inv;
702
+ progress = p;
703
+ break;
704
+ }
705
+ }
706
+
707
+ if (!foundInv || !progress) {
708
+ console.log(`\nāŒ Question ${questionId} not found in any investigation.\n`);
709
+ return;
710
+ }
711
+
712
+ // Parse analysis.md to extract category
713
+ const analysisPath = join(this.investigationsDir, foundInv, questionId, 'analysis.md');
714
+ let category: FailureCategory | undefined;
715
+
716
+ if (existsSync(analysisPath)) {
717
+ const content = await readFile(analysisPath, 'utf-8');
718
+ category = this.extractCategory(content);
719
+ }
720
+
721
+ // Update progress
722
+ progress.questions[questionId] = {
723
+ status: 'investigated',
724
+ category,
725
+ investigatedAt: new Date().toISOString(),
726
+ };
727
+
728
+ progress.investigated = Object.values(progress.questions).filter(q => q.status !== 'pending').length;
729
+
730
+ // Save progress
731
+ const progressPath = join(this.investigationsDir, foundInv, 'progress.json');
732
+ await writeFile(progressPath, JSON.stringify(progress, null, 2));
733
+
734
+ const remaining = progress.totalFailed - progress.investigated;
735
+ console.log(`\nāœ… Marked ${questionId} as investigated`);
736
+ if (category) {
737
+ console.log(` Category: ${category}`);
738
+ }
739
+ console.log(` Progress: ${progress.investigated}/${progress.totalFailed}`);
740
+ console.log(` Remaining: ${remaining}`);
741
+
742
+ if (remaining > 0) {
743
+ console.log(`\n Run: pnpm investigate --next\n`);
744
+ } else {
745
+ console.log(`\nšŸŽ‰ All questions investigated!`);
746
+ console.log(` Run: pnpm investigate --sync to sync fixes to dataset\n`);
747
+ }
748
+ }
749
+
750
+ private async markFixed(questionId: string): Promise<void> {
751
+ const investigations = await this.listInvestigations();
752
+
753
+ if (investigations.length === 0) {
754
+ console.log('\nšŸ“­ No investigations found.\n');
755
+ return;
756
+ }
757
+
758
+ // Find investigation containing this question
759
+ let foundInv: string | null = null;
760
+ let progress: InvestigationProgress | null = null;
761
+
762
+ for (const inv of investigations) {
763
+ const p = await this.loadProgress(inv);
764
+ if (p && questionId in p.questions) {
765
+ foundInv = inv;
766
+ progress = p;
767
+ break;
768
+ }
769
+ }
770
+
771
+ if (!foundInv || !progress) {
772
+ console.log(`\nāŒ Question ${questionId} not found in any investigation.\n`);
773
+ return;
774
+ }
775
+
776
+ const currentStatus = progress.questions[questionId]?.status;
777
+ if (currentStatus === 'pending') {
778
+ console.log(`\nāš ļø Question ${questionId} hasn't been investigated yet.`);
779
+ console.log(` Run: pnpm investigate --done ${questionId} first\n`);
780
+ return;
781
+ }
782
+
783
+ // Update status to fix-implemented
784
+ progress.questions[questionId] = {
785
+ ...progress.questions[questionId],
786
+ status: 'fix-implemented',
787
+ };
788
+
789
+ // Save progress
790
+ const progressPath = join(this.investigationsDir, foundInv, 'progress.json');
791
+ await writeFile(progressPath, JSON.stringify(progress, null, 2));
792
+
793
+ const fixedCount = Object.values(progress.questions).filter(
794
+ q => q.status === 'fix-implemented' || q.status === 'synced',
795
+ ).length;
796
+
797
+ console.log(`\nāœ… Marked ${questionId} as fix-implemented`);
798
+ console.log(` Fixed: ${fixedCount}/${progress.totalFailed}`);
799
+ console.log(`\n When ready, run: pnpm investigate --sync\n`);
800
+ }
801
+
802
+ // --------------------------------------------------------------------------
803
+ // Sync to Dataset
804
+ // --------------------------------------------------------------------------
805
+
806
+ private async syncToDataset(): Promise<void> {
807
+ const investigations = await this.listInvestigations();
808
+
809
+ if (investigations.length === 0) {
810
+ console.log('\nšŸ“­ No investigations found.\n');
811
+ return;
812
+ }
813
+
814
+ // Use most recent investigation
815
+ const latestInv = investigations[investigations.length - 1];
816
+ const progress = await this.loadProgress(latestInv);
817
+
818
+ if (!progress) {
819
+ console.log('\nāŒ Could not load progress.\n');
820
+ return;
821
+ }
822
+
823
+ // Load dataset
824
+ const datasetPath = join(this.datasetDir, `${progress.dataset}.json`);
825
+ if (!existsSync(datasetPath)) {
826
+ console.log(`\nāŒ Dataset not found: ${datasetPath}\n`);
827
+ return;
828
+ }
829
+
830
+ const dataset = JSON.parse(await readFile(datasetPath, 'utf-8')) as LongMemEvalQuestion[];
831
+ const datasetMap = new Map<string, LongMemEvalQuestion>();
832
+ for (const q of dataset) {
833
+ datasetMap.set(q.question_id, q);
834
+ }
835
+
836
+ let synced = 0;
837
+ let skipped = 0;
838
+
839
+ console.log(`\nšŸ”„ Syncing fixes to ${progress.dataset}.json...\n`);
840
+
841
+ for (const [questionId, qProgress] of Object.entries(progress.questions)) {
842
+ if (qProgress.status === 'pending') {
843
+ continue;
844
+ }
845
+
846
+ const analysisPath = join(this.investigationsDir, latestInv, questionId, 'analysis.md');
847
+ if (!existsSync(analysisPath)) {
848
+ continue;
849
+ }
850
+
851
+ const content = await readFile(analysisPath, 'utf-8');
852
+ const fixes = this.extractFixes(content);
853
+
854
+ if (!fixes.improved_question && !fixes.improved_answer && !fixes.improvement_note) {
855
+ skipped++;
856
+ continue;
857
+ }
858
+
859
+ const question = datasetMap.get(questionId);
860
+ if (!question) {
861
+ console.warn(` āš ļø Question ${questionId} not found in dataset`);
862
+ continue;
863
+ }
864
+
865
+ // Update question
866
+ let updated = false;
867
+ if (fixes.improved_question && fixes.improved_question !== question.improved_question) {
868
+ question.improved_question = fixes.improved_question;
869
+ updated = true;
870
+ }
871
+ if (fixes.improved_answer && fixes.improved_answer !== question.improved_answer) {
872
+ question.improved_answer = fixes.improved_answer;
873
+ updated = true;
874
+ }
875
+ if (fixes.improvement_note && fixes.improvement_note !== question.improvement_note) {
876
+ question.improvement_note = fixes.improvement_note;
877
+ updated = true;
878
+ }
879
+
880
+ if (updated) {
881
+ console.log(` āœ“ ${questionId}`);
882
+ synced++;
883
+
884
+ // Update progress status
885
+ progress.questions[questionId].status = 'synced';
886
+ } else {
887
+ skipped++;
888
+ }
889
+ }
890
+
891
+ // Save dataset with 4-space indentation
892
+ await writeFile(datasetPath, JSON.stringify(dataset, null, 4));
893
+
894
+ // Save progress
895
+ const progressPath = join(this.investigationsDir, latestInv, 'progress.json');
896
+ await writeFile(progressPath, JSON.stringify(progress, null, 2));
897
+
898
+ console.log(`\nāœ… Sync complete!`);
899
+ console.log(` Synced: ${synced}`);
900
+ console.log(` Skipped: ${skipped} (no changes or pending)`);
901
+ console.log(`\nšŸ’” Don't forget to run: pnpm run sync-improved-om-qa\n`);
902
+ }
903
+
904
+ // --------------------------------------------------------------------------
905
+ // Helpers
906
+ // --------------------------------------------------------------------------
907
+
908
+ private async findFailures(runIdOrPath: string): Promise<{ failuresPath: string; failures: FailuresFile }> {
909
+ // Check if it's a direct path
910
+ if (existsSync(runIdOrPath)) {
911
+ const content = await readFile(runIdOrPath, 'utf-8');
912
+ return { failuresPath: runIdOrPath, failures: JSON.parse(content) };
913
+ }
914
+
915
+ // Search in results directory
916
+ const configs = await readdir(this.resultsDir).catch(() => []);
917
+
918
+ for (const config of configs) {
919
+ const configDir = join(this.resultsDir, config);
920
+ const runs = await readdir(configDir).catch(() => []);
921
+
922
+ for (const run of runs) {
923
+ if (run === runIdOrPath || run.includes(runIdOrPath)) {
924
+ const failuresPath = join(configDir, run, 'failures.json');
925
+ if (existsSync(failuresPath)) {
926
+ const content = await readFile(failuresPath, 'utf-8');
927
+ return { failuresPath, failures: JSON.parse(content) };
928
+ }
929
+ }
930
+ }
931
+ }
932
+
933
+ throw new Error(`Could not find failures.json for: ${runIdOrPath}`);
934
+ }
935
+
936
+ private async listInvestigations(): Promise<string[]> {
937
+ if (!existsSync(this.investigationsDir)) {
938
+ return [];
939
+ }
940
+
941
+ const entries = await readdir(this.investigationsDir);
942
+ const investigations: string[] = [];
943
+
944
+ for (const entry of entries) {
945
+ const progressPath = join(this.investigationsDir, entry, 'progress.json');
946
+ if (existsSync(progressPath)) {
947
+ investigations.push(entry);
948
+ }
949
+ }
950
+
951
+ return investigations.sort();
952
+ }
953
+
954
+ private async loadProgress(investigation: string): Promise<InvestigationProgress | null> {
955
+ const progressPath = join(this.investigationsDir, investigation, 'progress.json');
956
+ if (!existsSync(progressPath)) {
957
+ return null;
958
+ }
959
+
960
+ return JSON.parse(await readFile(progressPath, 'utf-8'));
961
+ }
962
+
963
+ private extractCategory(content: string): FailureCategory | undefined {
964
+ const categoryMap: Record<string, FailureCategory> = {
965
+ 'Observer missed critical information': 'observer-miss',
966
+ 'Reflector lost/merged information incorrectly': 'reflector-loss',
967
+ 'Agent reasoning error': 'agent-reasoning',
968
+ 'Ambiguous/poorly-worded question': 'dataset-error', // Ambiguous questions are dataset errors
969
+ 'Dataset inconsistency/error': 'dataset-error',
970
+ 'RAG retrieval miss': 'rag-miss',
971
+ };
972
+
973
+ for (const [text, category] of Object.entries(categoryMap)) {
974
+ // Look for checked checkbox
975
+ const pattern = new RegExp(`\\[x\\]\\s*${text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}`, 'i');
976
+ if (pattern.test(content)) {
977
+ return category;
978
+ }
979
+ }
980
+
981
+ // Check for "Other"
982
+ if (/\[x\]\s*Other:/i.test(content)) {
983
+ return 'other';
984
+ }
985
+
986
+ return undefined;
987
+ }
988
+
989
+ private extractFixes(content: string): {
990
+ improved_question?: string;
991
+ improved_answer?: string;
992
+ improvement_note?: string;
993
+ } {
994
+ const fixes: {
995
+ improved_question?: string;
996
+ improved_answer?: string;
997
+ improvement_note?: string;
998
+ } = {};
999
+
1000
+ // Extract improved_question
1001
+ const iqMatch = content.match(/\*\*improved_question\*\*:\s*(.+?)(?:\n|$)/);
1002
+ if (iqMatch && iqMatch[1].trim() && !iqMatch[1].trim().startsWith('<!--')) {
1003
+ fixes.improved_question = iqMatch[1].trim();
1004
+ }
1005
+
1006
+ // Extract improved_answer
1007
+ const iaMatch = content.match(/\*\*improved_answer\*\*:\s*(.+?)(?:\n|$)/);
1008
+ if (iaMatch && iaMatch[1].trim() && !iaMatch[1].trim().startsWith('<!--')) {
1009
+ fixes.improved_answer = iaMatch[1].trim();
1010
+ }
1011
+
1012
+ // Extract improvement_note
1013
+ const inMatch = content.match(/\*\*improvement_note\*\*:\s*(.+?)(?:\n|$)/);
1014
+ if (inMatch && inMatch[1].trim() && !inMatch[1].trim().startsWith('<!--')) {
1015
+ fixes.improvement_note = inMatch[1].trim();
1016
+ }
1017
+
1018
+ return fixes;
1019
+ }
1020
+
1021
+ // --------------------------------------------------------------------------
1022
+ // Investigation Utilities
1023
+ // --------------------------------------------------------------------------
1024
+
1025
+ /**
1026
+ * Inspect a question's data - shows observations, messages, and metadata
1027
+ */
1028
+ private async inspectQuestion(questionId: string): Promise<void> {
1029
+ console.log(`\nšŸ” Inspecting question: ${questionId}\n`);
1030
+
1031
+ // Find the question in prepared data
1032
+ const { omPath, config, dataset } = await this.findPreparedData(questionId);
1033
+
1034
+ if (!omPath) {
1035
+ console.log(`āŒ No prepared data found for question ${questionId}`);
1036
+ return;
1037
+ }
1038
+
1039
+ console.log(`šŸ“ Found in: ${dataset}/${config}`);
1040
+
1041
+ // Load using storage adapter
1042
+ const storage = new PersistableInMemoryMemory({ readOnly: true });
1043
+ await storage.hydrate(omPath);
1044
+ const stats = storage.getStats();
1045
+
1046
+ console.log(`\nšŸ“Š Storage Stats:`);
1047
+ console.log(` Threads: ${stats.threads}`);
1048
+ console.log(` Messages: ${stats.messages}`);
1049
+ console.log(` OM Records: ${stats.observationalMemoryRecords}`);
1050
+
1051
+ // Get OM records - use resource scope (null threadId)
1052
+ // Note: resourceId in storage is prefixed with "resource_"
1053
+ const resourceId = `resource_${questionId}`;
1054
+ const omRecords = await storage.getObservationalMemoryHistory(null, resourceId);
1055
+
1056
+ if (omRecords.length === 0) {
1057
+ console.log(`\nāš ļø No observational memory records found`);
1058
+ return;
1059
+ }
1060
+
1061
+ const latestRecord = omRecords[0]; // Most recent is first
1062
+
1063
+ console.log(`\nšŸ“ Latest OM Record:`);
1064
+ console.log(` Created: ${latestRecord.createdAt}`);
1065
+ console.log(` Last Observed: ${latestRecord.lastObservedAt}`);
1066
+
1067
+ // Show observation stats
1068
+ const observations = latestRecord.activeObservations || '';
1069
+ const lines = observations.split('\n').filter((l: string) => l.trim());
1070
+ const dateHeaders = lines.filter((l: string) => /^Date:/i.test(l.trim()));
1071
+ const threadHeaders = lines.filter((l: string) => /<thread\s+id=/i.test(l));
1072
+
1073
+ console.log(`\nšŸ“‹ Observations:`);
1074
+ console.log(` Total lines: ${lines.length}`);
1075
+ console.log(` Date groups: ${dateHeaders.length}`);
1076
+ console.log(` Thread sections: ${threadHeaders.length}`);
1077
+
1078
+ // Show first few and last few lines
1079
+ console.log(`\n First 5 lines:`);
1080
+ lines.slice(0, 5).forEach((l: string) => console.log(` ${l.substring(0, 100)}${l.length > 100 ? '...' : ''}`));
1081
+
1082
+ if (lines.length > 10) {
1083
+ console.log(`\n Last 5 lines:`);
1084
+ lines.slice(-5).forEach((l: string) => console.log(` ${l.substring(0, 100)}${l.length > 100 ? '...' : ''}`));
1085
+ }
1086
+
1087
+ console.log(`\nšŸ’” Use --search to find specific content:`);
1088
+ console.log(` pnpm investigate --search "sneaker" -q ${questionId}`);
1089
+ }
1090
+
1091
+ /**
1092
+ * Search observations for keywords
1093
+ */
1094
+ private async searchObservations(questionId: string, keyword: string): Promise<void> {
1095
+ console.log(`\nšŸ” Searching for "${keyword}" in question ${questionId}\n`);
1096
+
1097
+ const { omPath } = await this.findPreparedData(questionId);
1098
+
1099
+ if (!omPath) {
1100
+ console.log(`āŒ No prepared data found for question ${questionId}`);
1101
+ return;
1102
+ }
1103
+
1104
+ // Load using storage adapter
1105
+ const storage = new PersistableInMemoryMemory({ readOnly: true });
1106
+ await storage.hydrate(omPath);
1107
+
1108
+ // Note: resourceId in storage is prefixed with "resource_"
1109
+ const resourceId = `resource_${questionId}`;
1110
+
1111
+ // Search in observations
1112
+ const omRecords = await storage.getObservationalMemoryHistory(null, resourceId);
1113
+ const latestRecord = omRecords[0];
1114
+ const observations = latestRecord?.activeObservations || '';
1115
+
1116
+ const keywordLower = keyword.toLowerCase();
1117
+ const obsLines = observations.split('\n');
1118
+ const obsMatches = obsLines
1119
+ .map((line: string, idx: number) => ({ line, idx }))
1120
+ .filter(({ line }: { line: string }) => line.toLowerCase().includes(keywordLower));
1121
+
1122
+ console.log(`šŸ“‹ Observations (${obsMatches.length} matches):`);
1123
+ if (obsMatches.length === 0) {
1124
+ console.log(` No matches found in observations`);
1125
+ } else {
1126
+ for (const { line, idx } of obsMatches) {
1127
+ // Highlight the keyword
1128
+ const highlighted = line.replace(new RegExp(`(${keyword})`, 'gi'), '\x1b[33m$1\x1b[0m');
1129
+ console.log(` [${idx}] ${highlighted}`);
1130
+ }
1131
+ }
1132
+
1133
+ // Search in raw messages
1134
+ const messagesResult = await storage.listMessagesByResourceId({ resourceId, perPage: false });
1135
+ const messages = messagesResult.messages;
1136
+ const msgMatches = messages.filter((m: any) => {
1137
+ const content = typeof m.content === 'string' ? m.content : JSON.stringify(m.content);
1138
+ return content.toLowerCase().includes(keywordLower);
1139
+ });
1140
+
1141
+ console.log(`\nšŸ’¬ Raw Messages (${msgMatches.length} matches):`);
1142
+ if (msgMatches.length === 0) {
1143
+ console.log(` No matches found in raw messages`);
1144
+ } else {
1145
+ for (const msg of msgMatches.slice(0, 10)) {
1146
+ const content = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content);
1147
+ const highlighted = content.replace(new RegExp(`(${keyword})`, 'gi'), '\x1b[33m$1\x1b[0m');
1148
+ const date = msg.createdAt ? new Date(msg.createdAt).toISOString().split('T')[0] : 'unknown';
1149
+ console.log(` [${date}] ${msg.role}: ${highlighted.substring(0, 120)}${content.length > 120 ? '...' : ''}`);
1150
+ }
1151
+ if (msgMatches.length > 10) {
1152
+ console.log(` ... and ${msgMatches.length - 10} more`);
1153
+ }
1154
+ }
1155
+
1156
+ // Summary
1157
+ const inObs = obsMatches.length > 0;
1158
+ const inRaw = msgMatches.length > 0;
1159
+
1160
+ console.log(`\nšŸ“Š Summary:`);
1161
+ if (inRaw && !inObs) {
1162
+ console.log(` āš ļø Found in raw messages but NOT in observations`);
1163
+ console.log(` → Likely an Observer miss`);
1164
+ } else if (inObs && inRaw) {
1165
+ console.log(` āœ“ Found in both raw messages and observations`);
1166
+ } else if (!inRaw && !inObs) {
1167
+ console.log(` āŒ Not found in raw data or observations`);
1168
+ }
1169
+ }
1170
+
1171
+ /**
1172
+ * Search original dataset for a keyword and show full context
1173
+ */
1174
+ private async searchOriginalDataset(questionId: string, keyword: string): Promise<void> {
1175
+ console.log(`\nšŸ” Searching original dataset for "${keyword}" in question ${questionId}\n`);
1176
+
1177
+ const loader = new DatasetLoader(this.datasetDir);
1178
+ const dataset = await loader.loadDataset('longmemeval_s');
1179
+ const question = dataset.find(q => q.question_id === questionId);
1180
+
1181
+ if (!question?.haystack_sessions || !question?.haystack_dates) {
1182
+ console.log(`āŒ Question not found or has no sessions`);
1183
+ return;
1184
+ }
1185
+
1186
+ const keywordLower = keyword.toLowerCase();
1187
+ let totalMatches = 0;
1188
+
1189
+ for (let sessionIdx = 0; sessionIdx < question.haystack_sessions.length; sessionIdx++) {
1190
+ const session = question.haystack_sessions[sessionIdx];
1191
+ const date = question.haystack_dates[sessionIdx] || 'unknown';
1192
+
1193
+ for (let turnIdx = 0; turnIdx < session.length; turnIdx++) {
1194
+ const turn = session[turnIdx];
1195
+ const content = turn.content || '';
1196
+
1197
+ if (content.toLowerCase().includes(keywordLower)) {
1198
+ totalMatches++;
1199
+ const roleColor = turn.role === 'user' ? '\x1b[36m' : '\x1b[90m';
1200
+ const reset = '\x1b[0m';
1201
+ const yellow = '\x1b[33m';
1202
+
1203
+ // Highlight the keyword
1204
+ const highlighted = content.replace(new RegExp(`(${keyword})`, 'gi'), `${yellow}$1${reset}`);
1205
+
1206
+ console.log(`${'─'.repeat(70)}`);
1207
+ console.log(`šŸ“… Session ${sessionIdx}, Turn ${turnIdx} | ${date}`);
1208
+ console.log(`${roleColor}[${turn.role.toUpperCase()}]:${reset}`);
1209
+ console.log(highlighted);
1210
+ console.log();
1211
+ }
1212
+ }
1213
+ }
1214
+
1215
+ if (totalMatches === 0) {
1216
+ console.log(`āŒ No matches found for "${keyword}" in original dataset`);
1217
+ } else {
1218
+ console.log(`${'─'.repeat(70)}`);
1219
+ console.log(`\nšŸ“Š Total: ${totalMatches} matches across ${question.haystack_sessions.length} sessions`);
1220
+ }
1221
+ }
1222
+
1223
+ /**
1224
+ * Trace information flow for a keyword through the pipeline
1225
+ */
1226
+ private async traceInformation(questionId: string, keyword: string): Promise<void> {
1227
+ console.log(`\nšŸ” Tracing "${keyword}" through pipeline for ${questionId}\n`);
1228
+
1229
+ const { omPath, config, dataset, questionDir } = await this.findPreparedData(questionId);
1230
+
1231
+ if (!omPath) {
1232
+ console.log(`āŒ No prepared data found for question ${questionId}`);
1233
+ return;
1234
+ }
1235
+
1236
+ const keywordLower = keyword.toLowerCase();
1237
+
1238
+ // 1. Check original dataset
1239
+ console.log(`\n1ļøāƒ£ Original Dataset:`);
1240
+ const loader = new DatasetLoader(this.datasetDir);
1241
+ const datasetName = dataset as 'longmemeval_s' | 'longmemeval_m';
1242
+ const questions = await loader.loadDataset(datasetName);
1243
+ const question = questions.find(q => q.question_id === questionId);
1244
+
1245
+ if (question) {
1246
+ // Search in haystack sessions
1247
+ let sessionMatches = 0;
1248
+ const matchingSessions: { idx: number; turnIdx: number; role: string; preview: string }[] = [];
1249
+
1250
+ if (question.haystack_sessions) {
1251
+ for (let sIdx = 0; sIdx < question.haystack_sessions.length; sIdx++) {
1252
+ const session = question.haystack_sessions[sIdx];
1253
+ for (let tIdx = 0; tIdx < session.length; tIdx++) {
1254
+ const turn = session[tIdx];
1255
+ if (turn.content.toLowerCase().includes(keywordLower)) {
1256
+ sessionMatches++;
1257
+ if (matchingSessions.length < 5) {
1258
+ matchingSessions.push({
1259
+ idx: sIdx,
1260
+ turnIdx: tIdx,
1261
+ role: turn.role,
1262
+ preview: turn.content.substring(0, 100),
1263
+ });
1264
+ }
1265
+ }
1266
+ }
1267
+ }
1268
+ }
1269
+
1270
+ console.log(` Found in ${sessionMatches} session turns`);
1271
+ for (const m of matchingSessions) {
1272
+ const highlighted = m.preview.replace(new RegExp(`(${keyword})`, 'gi'), '\x1b[33m$1\x1b[0m');
1273
+ console.log(` Session ${m.idx}, Turn ${m.turnIdx} (${m.role}): ${highlighted}...`);
1274
+ }
1275
+ if (sessionMatches > 5) {
1276
+ console.log(` ... and ${sessionMatches - 5} more`);
1277
+ }
1278
+ }
1279
+
1280
+ // 2. Check stored messages
1281
+ console.log(`\n2ļøāƒ£ Stored Messages (om.json):`);
1282
+ const storage = new PersistableInMemoryMemory({ readOnly: true });
1283
+ await storage.hydrate(omPath);
1284
+
1285
+ // Note: resourceId in storage is prefixed with "resource_"
1286
+ const resourceId = `resource_${questionId}`;
1287
+
1288
+ const messagesResult = await storage.listMessagesByResourceId({ resourceId, perPage: false });
1289
+ const allMessages = messagesResult.messages;
1290
+ const msgMatches = allMessages.filter((m: any) => {
1291
+ const content = typeof m.content === 'string' ? m.content : JSON.stringify(m.content);
1292
+ return content.toLowerCase().includes(keywordLower);
1293
+ });
1294
+
1295
+ console.log(` Found in ${msgMatches.length} messages`);
1296
+ for (const msg of msgMatches.slice(0, 3)) {
1297
+ const content = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content);
1298
+ const date = msg.createdAt ? new Date(msg.createdAt).toISOString() : 'unknown';
1299
+ console.log(` [${date}] ${msg.role}: ${content.substring(0, 80)}...`);
1300
+ }
1301
+
1302
+ // 3. Check observations
1303
+ console.log(`\n3ļøāƒ£ Observations (activeObservations):`);
1304
+ const omRecords = await storage.getObservationalMemoryHistory(null, resourceId);
1305
+ const latestRecord = omRecords[0]; // Most recent is first
1306
+ const observations = latestRecord?.activeObservations || '';
1307
+
1308
+ const obsLines = observations.split('\n');
1309
+ const obsMatches = obsLines.filter((l: string) => l.toLowerCase().includes(keywordLower));
1310
+
1311
+ console.log(` Found in ${obsMatches.length} observation lines`);
1312
+ for (const line of obsMatches.slice(0, 5)) {
1313
+ const highlighted = line.replace(new RegExp(`(${keyword})`, 'gi'), '\x1b[33m$1\x1b[0m');
1314
+ console.log(` ${highlighted.substring(0, 100)}...`);
1315
+ }
1316
+
1317
+ // 4. Check agent context (om.md)
1318
+ console.log(`\n4ļøāƒ£ Agent Context (om.md):`);
1319
+ const omMdPath = join(questionDir, 'om.md');
1320
+ if (existsSync(omMdPath)) {
1321
+ const omMd = await readFile(omMdPath, 'utf-8');
1322
+ const mdLines = omMd.split('\n');
1323
+ const mdMatches = mdLines.filter(l => l.toLowerCase().includes(keywordLower));
1324
+
1325
+ console.log(` Found in ${mdMatches.length} lines`);
1326
+ for (const line of mdMatches.slice(0, 5)) {
1327
+ const highlighted = line.replace(new RegExp(`(${keyword})`, 'gi'), '\x1b[33m$1\x1b[0m');
1328
+ console.log(` ${highlighted.substring(0, 100)}...`);
1329
+ }
1330
+ } else {
1331
+ console.log(` om.md not found (run benchmark first)`);
1332
+ }
1333
+
1334
+ // 5. Diagnosis
1335
+ console.log(`\nšŸ“Š Diagnosis:`);
1336
+ const inDataset =
1337
+ question && question.haystack_sessions?.some(s => s.some(t => t.content.toLowerCase().includes(keywordLower)));
1338
+ const inMessages = msgMatches.length > 0;
1339
+ const inObservations = obsMatches.length > 0;
1340
+
1341
+ if (inDataset && !inMessages) {
1342
+ console.log(` āŒ Lost during message storage - check prepare.ts`);
1343
+ } else if (inMessages && !inObservations) {
1344
+ console.log(` āŒ Observer missed this information`);
1345
+ console.log(` → Check Observer prompts or message batching`);
1346
+ } else if (inObservations) {
1347
+ console.log(` āœ“ Information preserved through pipeline`);
1348
+ console.log(` → If agent still failed, it's a reasoning error`);
1349
+ } else if (!inDataset) {
1350
+ console.log(` āš ļø Not found in original dataset`);
1351
+ }
1352
+ }
1353
+
1354
+ // --------------------------------------------------------------------------
1355
+ // Date-Based Observation Viewer
1356
+ // --------------------------------------------------------------------------
1357
+
1358
+ /**
1359
+ * View observations and raw messages around a specific date
1360
+ */
1361
+ private async viewObservationsAroundDate(
1362
+ questionId: string,
1363
+ dateStr: string,
1364
+ contextDays: number = 1,
1365
+ ): Promise<void> {
1366
+ console.log(`\nšŸ—“ļø Viewing data around "${dateStr}" for question ${questionId}\n`);
1367
+
1368
+ // Parse the target date
1369
+ const targetDate = this.parseFlexibleDate(dateStr);
1370
+ if (!targetDate) {
1371
+ console.log(`āŒ Could not parse date: "${dateStr}"`);
1372
+ console.log(` Try formats like: "2023/05/29", "May 29, 2023", "05-29"`);
1373
+ return;
1374
+ }
1375
+
1376
+ const startDate = new Date(targetDate);
1377
+ startDate.setDate(startDate.getDate() - contextDays);
1378
+ const endDate = new Date(targetDate);
1379
+ endDate.setDate(endDate.getDate() + contextDays);
1380
+
1381
+ console.log(`šŸ“… Target: ${targetDate.toDateString()}`);
1382
+ console.log(`šŸ“… Range: ${startDate.toDateString()} to ${endDate.toDateString()}\n`);
1383
+
1384
+ // Find prepared data
1385
+ const { omPath, questionDir } = await this.findPreparedData(questionId);
1386
+ if (!omPath) {
1387
+ console.log(`āŒ No prepared data found for question ${questionId}`);
1388
+ return;
1389
+ }
1390
+
1391
+ // Load storage
1392
+ const storage = new PersistableInMemoryMemory({ readOnly: true });
1393
+ await storage.hydrate(omPath);
1394
+ const resourceId = `resource_${questionId}`;
1395
+
1396
+ // 1. Show raw messages in date range
1397
+ console.log(`1ļøāƒ£ Raw Messages in Range:`);
1398
+ console.log(`${'─'.repeat(60)}`);
1399
+
1400
+ const messagesResult = await storage.listMessagesByResourceId({ resourceId, perPage: false });
1401
+ const allMessages = messagesResult.messages;
1402
+
1403
+ const messagesInRange = allMessages
1404
+ .filter((m: any) => {
1405
+ if (!m.createdAt) return false;
1406
+ const msgDate = new Date(m.createdAt);
1407
+ return msgDate >= startDate && msgDate <= endDate;
1408
+ })
1409
+ .sort((a: any, b: any) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
1410
+
1411
+ if (messagesInRange.length === 0) {
1412
+ console.log(` No messages found in this date range`);
1413
+ } else {
1414
+ console.log(` Found ${messagesInRange.length} messages\n`);
1415
+ for (const msg of messagesInRange) {
1416
+ const content = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content);
1417
+ const date = new Date(msg.createdAt);
1418
+ const dateStr = date.toLocaleDateString('en-US', {
1419
+ month: 'short',
1420
+ day: 'numeric',
1421
+ hour: '2-digit',
1422
+ minute: '2-digit',
1423
+ });
1424
+ const isTargetDay = date.toDateString() === targetDate.toDateString();
1425
+ const prefix = isTargetDay ? '→ ' : ' ';
1426
+ const highlight = isTargetDay ? '\x1b[33m' : '\x1b[90m';
1427
+ const reset = '\x1b[0m';
1428
+
1429
+ console.log(`${prefix}${highlight}[${dateStr}] ${msg.role}:${reset}`);
1430
+ console.log(`${prefix}${highlight}${content.substring(0, 200)}${content.length > 200 ? '...' : ''}${reset}\n`);
1431
+ }
1432
+ }
1433
+
1434
+ // 2. Show observations for dates in range
1435
+ console.log(`\n2ļøāƒ£ Observations in Range:`);
1436
+ console.log(`${'─'.repeat(60)}`);
1437
+
1438
+ const omRecords = await storage.getObservationalMemoryHistory(null, resourceId);
1439
+ const latestRecord = omRecords[0];
1440
+ const observations = latestRecord?.activeObservations || '';
1441
+
1442
+ if (!observations) {
1443
+ console.log(` No observations found`);
1444
+ } else {
1445
+ // Parse observations by date
1446
+ const lines = observations.split('\n');
1447
+ let currentDateGroup = '';
1448
+ let inRange = false;
1449
+ let observationsInRange: { date: string; lines: string[] }[] = [];
1450
+ let currentGroup: { date: string; lines: string[] } | null = null;
1451
+
1452
+ for (const line of lines) {
1453
+ // Check for date headers like "Date: May 29, 2023" or "Date: Monday, May 29, 2023"
1454
+ const dateMatch = line.match(/^Date:\s*(.+)$/i);
1455
+ if (dateMatch) {
1456
+ // Save previous group if in range
1457
+ if (currentGroup && currentGroup.lines.length > 0) {
1458
+ observationsInRange.push(currentGroup);
1459
+ }
1460
+
1461
+ currentDateGroup = dateMatch[1].trim();
1462
+ const parsedDate = this.parseFlexibleDate(currentDateGroup);
1463
+ inRange = parsedDate ? parsedDate >= startDate && parsedDate <= endDate : false;
1464
+
1465
+ if (inRange) {
1466
+ currentGroup = { date: currentDateGroup, lines: [] };
1467
+ } else {
1468
+ currentGroup = null;
1469
+ }
1470
+ } else if (inRange && currentGroup && line.trim()) {
1471
+ // Skip thread tags but include observation content
1472
+ if (!line.match(/^<\/?thread/)) {
1473
+ currentGroup.lines.push(line);
1474
+ }
1475
+ }
1476
+ }
1477
+
1478
+ // Don't forget the last group
1479
+ if (currentGroup && currentGroup.lines.length > 0) {
1480
+ observationsInRange.push(currentGroup);
1481
+ }
1482
+
1483
+ if (observationsInRange.length === 0) {
1484
+ console.log(` No observations found for dates in range`);
1485
+ } else {
1486
+ for (const group of observationsInRange) {
1487
+ const parsedDate = this.parseFlexibleDate(group.date);
1488
+ const isTargetDay = parsedDate && parsedDate.toDateString() === targetDate.toDateString();
1489
+ const highlight = isTargetDay ? '\x1b[33m' : '\x1b[0m';
1490
+ const reset = '\x1b[0m';
1491
+
1492
+ console.log(`\n${highlight}šŸ“… ${group.date}${reset}`);
1493
+ for (const line of group.lines) {
1494
+ console.log(` ${highlight}${line}${reset}`);
1495
+ }
1496
+ }
1497
+ }
1498
+ }
1499
+
1500
+ // 3. Show original dataset sessions for this date
1501
+ console.log(`\n\n3ļøāƒ£ Original Dataset Sessions:`);
1502
+ console.log(`${'─'.repeat(60)}`);
1503
+
1504
+ const loader = new DatasetLoader(this.datasetDir);
1505
+ const dataset = await loader.loadDataset('longmemeval_s');
1506
+ const question = dataset.find(q => q.question_id === questionId);
1507
+
1508
+ if (question?.haystack_sessions && question?.haystack_dates) {
1509
+ const sessionsInRange: { idx: number; date: string; turns: any[] }[] = [];
1510
+
1511
+ for (let i = 0; i < question.haystack_dates.length; i++) {
1512
+ const sessionDateStr = question.haystack_dates[i];
1513
+ const sessionDate = this.parseFlexibleDate(sessionDateStr);
1514
+
1515
+ if (sessionDate && sessionDate >= startDate && sessionDate <= endDate) {
1516
+ sessionsInRange.push({
1517
+ idx: i,
1518
+ date: sessionDateStr,
1519
+ turns: question.haystack_sessions[i] || [],
1520
+ });
1521
+ }
1522
+ }
1523
+
1524
+ if (sessionsInRange.length === 0) {
1525
+ console.log(` No sessions found in date range`);
1526
+ } else {
1527
+ console.log(` Found ${sessionsInRange.length} sessions\n`);
1528
+ for (const session of sessionsInRange) {
1529
+ const sessionDate = this.parseFlexibleDate(session.date);
1530
+ const isTargetDay = sessionDate && sessionDate.toDateString() === targetDate.toDateString();
1531
+ const highlight = isTargetDay ? '\x1b[33m' : '\x1b[90m';
1532
+ const reset = '\x1b[0m';
1533
+
1534
+ console.log(`${highlight}━━━ Session ${session.idx}: ${session.date} ━━━${reset}`);
1535
+ for (let t = 0; t < session.turns.length; t++) {
1536
+ const turn = session.turns[t];
1537
+ console.log(`${highlight}[Turn ${t}] ${turn.role}:${reset}`);
1538
+ console.log(
1539
+ `${highlight}${turn.content.substring(0, 300)}${turn.content.length > 300 ? '...' : ''}${reset}\n`,
1540
+ );
1541
+ }
1542
+ }
1543
+ }
1544
+ } else {
1545
+ console.log(` Question not found in dataset`);
1546
+ }
1547
+ }
1548
+
1549
+ // --------------------------------------------------------------------------
1550
+ // Session Viewer
1551
+ // --------------------------------------------------------------------------
1552
+
1553
+ /**
1554
+ * List all sessions with their dates for a question
1555
+ */
1556
+ private async listSessions(questionId: string): Promise<void> {
1557
+ console.log(`\nšŸ“‹ Sessions for question ${questionId}\n`);
1558
+
1559
+ const loader = new DatasetLoader(this.datasetDir);
1560
+ const dataset = await loader.loadDataset('longmemeval_s');
1561
+ const question = dataset.find(q => q.question_id === questionId);
1562
+
1563
+ if (!question?.haystack_sessions || !question?.haystack_dates) {
1564
+ console.log(`āŒ Question not found or has no sessions`);
1565
+ return;
1566
+ }
1567
+
1568
+ console.log(`Total sessions: ${question.haystack_sessions.length}\n`);
1569
+ console.log(`${'Idx'.padStart(4)} | ${'Date'.padEnd(25)} | Turns | Preview`);
1570
+ console.log(`${'─'.repeat(80)}`);
1571
+
1572
+ for (let i = 0; i < question.haystack_sessions.length; i++) {
1573
+ const session = question.haystack_sessions[i];
1574
+ const date = question.haystack_dates[i] || 'unknown';
1575
+ const turns = session.length;
1576
+ const firstUserTurn = session.find(t => t.role === 'user');
1577
+ const preview = firstUserTurn?.content.substring(0, 40) || '';
1578
+
1579
+ console.log(`${String(i).padStart(4)} | ${date.padEnd(25)} | ${String(turns).padStart(5)} | ${preview}...`);
1580
+ }
1581
+
1582
+ console.log(`\nUse --session <idx> -q ${questionId} to view a specific session`);
1583
+ }
1584
+
1585
+ /**
1586
+ * View a specific session from the original dataset
1587
+ */
1588
+ private async viewSession(questionId: string, sessionIdx: number): Promise<void> {
1589
+ console.log(`\nšŸ“„ Session ${sessionIdx} for question ${questionId}\n`);
1590
+
1591
+ const loader = new DatasetLoader(this.datasetDir);
1592
+ const dataset = await loader.loadDataset('longmemeval_s');
1593
+ const question = dataset.find(q => q.question_id === questionId);
1594
+
1595
+ if (!question?.haystack_sessions || !question?.haystack_dates) {
1596
+ console.log(`āŒ Question not found or has no sessions`);
1597
+ return;
1598
+ }
1599
+
1600
+ if (sessionIdx < 0 || sessionIdx >= question.haystack_sessions.length) {
1601
+ console.log(`āŒ Invalid session index. Valid range: 0-${question.haystack_sessions.length - 1}`);
1602
+ return;
1603
+ }
1604
+
1605
+ const session = question.haystack_sessions[sessionIdx];
1606
+ const date = question.haystack_dates[sessionIdx] || 'unknown';
1607
+
1608
+ console.log(`šŸ“… Date: ${date}`);
1609
+ console.log(`šŸ’¬ Turns: ${session.length}`);
1610
+ console.log(`${'─'.repeat(60)}\n`);
1611
+
1612
+ for (let t = 0; t < session.length; t++) {
1613
+ const turn = session[t];
1614
+ const roleColor = turn.role === 'user' ? '\x1b[36m' : '\x1b[90m';
1615
+ const reset = '\x1b[0m';
1616
+
1617
+ console.log(`${roleColor}[Turn ${t}] ${turn.role.toUpperCase()}:${reset}`);
1618
+ console.log(`${turn.content}\n`);
1619
+ }
1620
+
1621
+ // Also show what was observed for this date
1622
+ const { omPath } = await this.findPreparedData(questionId);
1623
+ if (omPath) {
1624
+ const storage = new PersistableInMemoryMemory({ readOnly: true });
1625
+ await storage.hydrate(omPath);
1626
+ const resourceId = `resource_${questionId}`;
1627
+
1628
+ const omRecords = await storage.getObservationalMemoryHistory(null, resourceId);
1629
+ const latestRecord = omRecords[0];
1630
+ const observations = latestRecord?.activeObservations || '';
1631
+
1632
+ if (observations) {
1633
+ // Find observations for this date
1634
+ const sessionDate = this.parseFlexibleDate(date);
1635
+ if (sessionDate) {
1636
+ const dateStr = sessionDate.toLocaleDateString('en-US', {
1637
+ weekday: 'long',
1638
+ month: 'long',
1639
+ day: 'numeric',
1640
+ year: 'numeric',
1641
+ });
1642
+
1643
+ console.log(`\n${'─'.repeat(60)}`);
1644
+ console.log(`\nšŸ” Observations extracted for ${dateStr}:\n`);
1645
+
1646
+ const lines = observations.split('\n');
1647
+ let inTargetDate = false;
1648
+ let foundAny = false;
1649
+
1650
+ for (const line of lines) {
1651
+ const dateMatch = line.match(/^Date:\s*(.+)$/i);
1652
+ if (dateMatch) {
1653
+ const lineDate = this.parseFlexibleDate(dateMatch[1].trim());
1654
+ inTargetDate = lineDate?.toDateString() === sessionDate.toDateString();
1655
+ } else if (inTargetDate && line.trim() && !line.match(/^<\/?thread/)) {
1656
+ console.log(` ${line}`);
1657
+ foundAny = true;
1658
+ }
1659
+ }
1660
+
1661
+ if (!foundAny) {
1662
+ console.log(` āš ļø No observations found for this date`);
1663
+ }
1664
+ }
1665
+ }
1666
+ }
1667
+ }
1668
+
1669
+ /**
1670
+ * Parse various date formats flexibly
1671
+ */
1672
+ private parseFlexibleDate(dateStr: string): Date | null {
1673
+ if (!dateStr) return null;
1674
+
1675
+ // Try various formats
1676
+ const formats = [
1677
+ // "2023/05/29 (Mon) 15:01" - LongMemEval format
1678
+ /^(\d{4})\/(\d{2})\/(\d{2})/,
1679
+ // "May 29, 2023" or "Monday, May 29, 2023"
1680
+ /(?:(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday),?\s*)?(\w+)\s+(\d{1,2}),?\s*(\d{4})/i,
1681
+ // "05-29-2023" or "05/29/2023"
1682
+ /^(\d{2})[-\/](\d{2})[-\/](\d{4})$/,
1683
+ // "2023-05-29"
1684
+ /^(\d{4})-(\d{2})-(\d{2})/,
1685
+ ];
1686
+
1687
+ // Try LongMemEval format first: "2023/05/29 (Mon) 15:01"
1688
+ const longMemMatch = dateStr.match(/^(\d{4})\/(\d{2})\/(\d{2})/);
1689
+ if (longMemMatch) {
1690
+ return new Date(parseInt(longMemMatch[1]), parseInt(longMemMatch[2]) - 1, parseInt(longMemMatch[3]));
1691
+ }
1692
+
1693
+ // Try "May 29, 2023" format
1694
+ const monthNameMatch = dateStr.match(
1695
+ /(?:(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday),?\s*)?(\w+)\s+(\d{1,2}),?\s*(\d{4})/i,
1696
+ );
1697
+ if (monthNameMatch) {
1698
+ const months: Record<string, number> = {
1699
+ january: 0,
1700
+ february: 1,
1701
+ march: 2,
1702
+ april: 3,
1703
+ may: 4,
1704
+ june: 5,
1705
+ july: 6,
1706
+ august: 7,
1707
+ september: 8,
1708
+ october: 9,
1709
+ november: 10,
1710
+ december: 11,
1711
+ };
1712
+ const monthNum = months[monthNameMatch[1].toLowerCase()];
1713
+ if (monthNum !== undefined) {
1714
+ return new Date(parseInt(monthNameMatch[3]), monthNum, parseInt(monthNameMatch[2]));
1715
+ }
1716
+ }
1717
+
1718
+ // Try ISO format: "2023-05-29"
1719
+ const isoMatch = dateStr.match(/^(\d{4})-(\d{2})-(\d{2})/);
1720
+ if (isoMatch) {
1721
+ return new Date(parseInt(isoMatch[1]), parseInt(isoMatch[2]) - 1, parseInt(isoMatch[3]));
1722
+ }
1723
+
1724
+ // Fallback to native Date parsing
1725
+ const parsed = new Date(dateStr);
1726
+ return isNaN(parsed.getTime()) ? null : parsed;
1727
+ }
1728
+
1729
+ // --------------------------------------------------------------------------
1730
+ // Stale Data Detection
1731
+ // --------------------------------------------------------------------------
1732
+
1733
+ /**
1734
+ * Check if prepared data is stale (missing per-thread cursors).
1735
+ * Data prepared before the per-thread cursor fix will be missing
1736
+ * lastObservedAt on thread metadata, which can cause observation gaps.
1737
+ */
1738
+ private async checkStaleData(questionId?: string, staleOnly?: boolean): Promise<void> {
1739
+ console.log('\nšŸ” Checking for stale prepared data (pre-cursor-fix)...\n');
1740
+
1741
+ interface StaleCheckResult {
1742
+ questionId: string;
1743
+ config: string;
1744
+ dataset: string;
1745
+ isStale: boolean;
1746
+ threadCount: number;
1747
+ threadsWithCursor: number;
1748
+ reason?: string;
1749
+ }
1750
+
1751
+ const results: StaleCheckResult[] = [];
1752
+
1753
+ // If specific question, just check that one
1754
+ if (questionId) {
1755
+ const { omPath, config, dataset } = await this.findPreparedData(questionId);
1756
+ if (!omPath) {
1757
+ console.log(`āŒ No prepared data found for question ${questionId}`);
1758
+ return;
1759
+ }
1760
+
1761
+ const result = await this.checkSingleQuestion(questionId, omPath, config, dataset);
1762
+ results.push(result);
1763
+ } else {
1764
+ // Check all questions in all configs
1765
+ const datasets = ['longmemeval_s', 'longmemeval_m'];
1766
+
1767
+ for (const dataset of datasets) {
1768
+ const datasetDir = join(this.preparedDataDir, dataset);
1769
+ if (!existsSync(datasetDir)) continue;
1770
+
1771
+ const configs = await readdir(datasetDir).catch(() => []);
1772
+
1773
+ for (const config of configs) {
1774
+ // Only check OM configs
1775
+ if (!config.includes('observational-memory') && !config.startsWith('om-')) continue;
1776
+
1777
+ const configDir = join(datasetDir, config);
1778
+ const questions = await readdir(configDir).catch(() => []);
1779
+
1780
+ for (const qId of questions) {
1781
+ const omPath = join(configDir, qId, 'om.json');
1782
+ if (!existsSync(omPath)) continue;
1783
+
1784
+ const result = await this.checkSingleQuestion(qId, omPath, config, dataset);
1785
+ results.push(result);
1786
+ }
1787
+ }
1788
+ }
1789
+ }
1790
+
1791
+ // Display results
1792
+ const staleResults = results.filter(r => r.isStale);
1793
+ const freshResults = results.filter(r => !r.isStale);
1794
+
1795
+ if (staleOnly) {
1796
+ // Only show stale questions
1797
+ if (staleResults.length === 0) {
1798
+ console.log('āœ… No stale data found!');
1799
+ return;
1800
+ }
1801
+
1802
+ console.log(`Found ${staleResults.length} stale question(s):\n`);
1803
+
1804
+ // Group by config
1805
+ const byConfig: Record<string, StaleCheckResult[]> = {};
1806
+ for (const r of staleResults) {
1807
+ const key = `${r.dataset}/${r.config}`;
1808
+ if (!byConfig[key]) byConfig[key] = [];
1809
+ byConfig[key].push(r);
1810
+ }
1811
+
1812
+ for (const [configKey, questions] of Object.entries(byConfig)) {
1813
+ console.log(`šŸ“ ${configKey} (${questions.length} stale):`);
1814
+ for (const q of questions.slice(0, 20)) {
1815
+ console.log(` ${q.questionId} - ${q.reason}`);
1816
+ }
1817
+ if (questions.length > 20) {
1818
+ console.log(` ... and ${questions.length - 20} more`);
1819
+ }
1820
+ console.log();
1821
+ }
1822
+
1823
+ // Output question IDs for re-preparation
1824
+ console.log('\nšŸ“‹ Stale question IDs (for re-preparation):');
1825
+ console.log(staleResults.map(r => r.questionId).join('\n'));
1826
+ } else {
1827
+ // Show summary
1828
+ console.log(`šŸ“Š Summary:`);
1829
+ console.log(` Total checked: ${results.length}`);
1830
+ console.log(` āœ… Fresh (has per-thread cursors): ${freshResults.length}`);
1831
+ console.log(` āš ļø Stale (missing cursors): ${staleResults.length}`);
1832
+
1833
+ if (staleResults.length > 0) {
1834
+ console.log(`\nāš ļø Stale questions may have observation gaps!`);
1835
+ console.log(` Run with --stale-only to list them for re-preparation.`);
1836
+ }
1837
+ }
1838
+ }
1839
+
1840
+ private async checkSingleQuestion(
1841
+ questionId: string,
1842
+ omPath: string,
1843
+ config: string,
1844
+ dataset: string,
1845
+ ): Promise<{
1846
+ questionId: string;
1847
+ config: string;
1848
+ dataset: string;
1849
+ isStale: boolean;
1850
+ threadCount: number;
1851
+ threadsWithCursor: number;
1852
+ reason?: string;
1853
+ }> {
1854
+ try {
1855
+ const data = JSON.parse(await readFile(omPath, 'utf-8'));
1856
+ const threads = data.threads || [];
1857
+
1858
+ let threadCount = 0;
1859
+ let threadsWithCursor = 0;
1860
+
1861
+ for (const threadPair of threads) {
1862
+ // Structure is [threadId, threadObject]
1863
+ if (!Array.isArray(threadPair) || threadPair.length < 2) continue;
1864
+
1865
+ const threadObj = threadPair[1];
1866
+ if (typeof threadObj !== 'object') continue;
1867
+
1868
+ threadCount++;
1869
+ const metadata = threadObj.metadata || {};
1870
+
1871
+ // Check both old location (metadata.lastObservedAt) and new location (metadata.mastra.om.lastObservedAt)
1872
+ const hasOldCursor = !!metadata.lastObservedAt;
1873
+ const hasNewCursor = !!metadata?.mastra?.om?.lastObservedAt;
1874
+
1875
+ if (hasOldCursor || hasNewCursor) {
1876
+ threadsWithCursor++;
1877
+ }
1878
+ }
1879
+
1880
+ const isStale = threadCount > 0 && threadsWithCursor === 0;
1881
+ const reason = isStale
1882
+ ? `0/${threadCount} threads have lastObservedAt`
1883
+ : threadsWithCursor < threadCount
1884
+ ? `${threadsWithCursor}/${threadCount} threads have lastObservedAt`
1885
+ : undefined;
1886
+
1887
+ return {
1888
+ questionId,
1889
+ config,
1890
+ dataset,
1891
+ isStale,
1892
+ threadCount,
1893
+ threadsWithCursor,
1894
+ reason,
1895
+ };
1896
+ } catch (error) {
1897
+ return {
1898
+ questionId,
1899
+ config,
1900
+ dataset,
1901
+ isStale: true,
1902
+ threadCount: 0,
1903
+ threadsWithCursor: 0,
1904
+ reason: `Error reading om.json: ${error}`,
1905
+ };
1906
+ }
1907
+ }
1908
+
1909
+ /**
1910
+ * Find prepared data for a question across all configs
1911
+ */
1912
+ private async findPreparedData(questionId: string): Promise<{
1913
+ omPath: string | null;
1914
+ config: string;
1915
+ dataset: string;
1916
+ questionDir: string;
1917
+ }> {
1918
+ const datasets = ['longmemeval_s', 'longmemeval_m'];
1919
+
1920
+ for (const dataset of datasets) {
1921
+ const datasetDir = join(this.preparedDataDir, dataset);
1922
+ if (!existsSync(datasetDir)) continue;
1923
+
1924
+ const configs = await readdir(datasetDir).catch(() => []);
1925
+
1926
+ for (const config of configs) {
1927
+ const questionDir = join(datasetDir, config, questionId);
1928
+ const omPath = join(questionDir, 'om.json');
1929
+
1930
+ if (existsSync(omPath)) {
1931
+ return { omPath, config, dataset, questionDir };
1932
+ }
1933
+ }
1934
+ }
1935
+
1936
+ return { omPath: null, config: '', dataset: '', questionDir: '' };
1937
+ }
1938
+
1939
+ // --------------------------------------------------------------------------
1940
+ // Add Improvement to Dataset
1941
+ // --------------------------------------------------------------------------
1942
+
1943
+ private async addImprovement(
1944
+ questionId: string,
1945
+ improvedQuestion?: string,
1946
+ improvedAnswer?: string,
1947
+ improvementNote?: string,
1948
+ category?: FailureCategory,
1949
+ clearImproved?: string | boolean,
1950
+ ): Promise<void> {
1951
+ if (!clearImproved && !improvedQuestion && !improvedAnswer && !improvementNote && !category) {
1952
+ console.log(
1953
+ 'āŒ At least one of --improve-question, --improve-answer, --improve-note, --category, or --clear-improved is required',
1954
+ );
1955
+ return;
1956
+ }
1957
+
1958
+ // Parse clearImproved into specific fields to clear
1959
+ const clearFields = new Set<string>();
1960
+ if (clearImproved === true || clearImproved === 'all') {
1961
+ clearFields.add('question');
1962
+ clearFields.add('answer');
1963
+ clearFields.add('note');
1964
+ clearFields.add('category');
1965
+ } else if (typeof clearImproved === 'string') {
1966
+ // Support comma-separated list: "note,category"
1967
+ for (const field of clearImproved.split(',')) {
1968
+ const trimmed = field.trim().toLowerCase();
1969
+ if (['question', 'answer', 'note', 'category'].includes(trimmed)) {
1970
+ clearFields.add(trimmed);
1971
+ } else {
1972
+ console.log(`āŒ Unknown field to clear: ${trimmed}. Valid fields: question, answer, note, category, all`);
1973
+ return;
1974
+ }
1975
+ }
1976
+ }
1977
+
1978
+ const datasetPath = join(this.datasetDir, 'longmemeval_s.json');
1979
+
1980
+ if (!existsSync(datasetPath)) {
1981
+ console.log(`āŒ Dataset not found: ${datasetPath}`);
1982
+ return;
1983
+ }
1984
+
1985
+ // Load dataset
1986
+ const datasetContent = await readFile(datasetPath, 'utf-8');
1987
+ const dataset: LongMemEvalQuestion[] = JSON.parse(datasetContent);
1988
+
1989
+ // Find the question
1990
+ const questionIndex = dataset.findIndex(q => q.question_id === questionId);
1991
+ if (questionIndex === -1) {
1992
+ console.log(`āŒ Question not found: ${questionId}`);
1993
+ return;
1994
+ }
1995
+
1996
+ const question = dataset[questionIndex];
1997
+
1998
+ // Show current state
1999
+ console.log(`\nšŸ“ Updating question: ${questionId}`);
2000
+ console.log(` Type: ${question.question_type}`);
2001
+ console.log(` Question: ${question.question.substring(0, 80)}...`);
2002
+ console.log(` Expected: ${String(question.answer).substring(0, 80)}`);
2003
+
2004
+ // Show what's being updated
2005
+ console.log('\nšŸ“‹ Changes:');
2006
+
2007
+ // Handle clearing specific fields
2008
+ if (clearFields.has('question') && question.improved_question) {
2009
+ console.log(` improved_question: ${question.improved_question} → (cleared)`);
2010
+ delete question.improved_question;
2011
+ }
2012
+ if (clearFields.has('answer') && question.improved_answer) {
2013
+ console.log(` improved_answer: ${question.improved_answer.substring(0, 50)}... → (cleared)`);
2014
+ delete question.improved_answer;
2015
+ }
2016
+ if (clearFields.has('note') && question.improvement_note) {
2017
+ console.log(` improvement_note: ${question.improvement_note.substring(0, 50)}... → (cleared)`);
2018
+ delete question.improvement_note;
2019
+ }
2020
+ if (clearFields.has('category') && question.failure_category) {
2021
+ console.log(` failure_category: ${question.failure_category} → (cleared)`);
2022
+ delete question.failure_category;
2023
+ }
2024
+
2025
+ // Handle setting new values (can be combined with clearing other fields)
2026
+ if (improvedQuestion) {
2027
+ console.log(` improved_question: ${question.improved_question || '(not set)'} → ${improvedQuestion}`);
2028
+ question.improved_question = improvedQuestion;
2029
+ }
2030
+
2031
+ if (improvedAnswer) {
2032
+ console.log(` improved_answer: ${question.improved_answer || '(not set)'} → ${improvedAnswer}`);
2033
+ question.improved_answer = improvedAnswer;
2034
+ }
2035
+
2036
+ if (improvementNote) {
2037
+ console.log(` improvement_note: ${question.improvement_note || '(not set)'} → ${improvementNote}`);
2038
+ question.improvement_note = improvementNote;
2039
+ }
2040
+
2041
+ if (category) {
2042
+ console.log(` failure_category: ${question.failure_category || '(not set)'} → ${category}`);
2043
+ question.failure_category = category;
2044
+ }
2045
+
2046
+ // Save dataset with 4-space indentation
2047
+ await writeFile(datasetPath, JSON.stringify(dataset, null, 4));
2048
+
2049
+ console.log('\nāœ… Dataset updated!');
2050
+ console.log(' Run `pnpm run sync-improved-om-qa` to sync to prepared data');
2051
+ }
2052
+
2053
+ // --------------------------------------------------------------------------
2054
+ // Check Duplicate Observations
2055
+ // --------------------------------------------------------------------------
2056
+
2057
+ private async checkDuplicateObservations(questionId?: string): Promise<void> {
2058
+ if (!questionId) {
2059
+ console.log('āŒ Please provide a question ID with -q <question-id>');
2060
+ return;
2061
+ }
2062
+
2063
+ console.log(`\nšŸ” Checking for duplicate observations in ${questionId}...\n`);
2064
+
2065
+ // Find the om.json file
2066
+ const configs = ['observational-memory', 'om-gpt5-mini', 'om-gpt5', 'om-gemini-3-pro', 'om-gemini-3-flash'];
2067
+ let omJsonPath: string | null = null;
2068
+ let foundConfig: string | null = null;
2069
+
2070
+ for (const config of configs) {
2071
+ const path = join(this.preparedDataDir, 'longmemeval_s', config, questionId, 'om.json');
2072
+ if (existsSync(path)) {
2073
+ omJsonPath = path;
2074
+ foundConfig = config;
2075
+ break;
2076
+ }
2077
+ }
2078
+
2079
+ if (!omJsonPath) {
2080
+ console.log(`āŒ Could not find om.json for question ${questionId}`);
2081
+ return;
2082
+ }
2083
+
2084
+ console.log(`šŸ“ Found data in config: ${foundConfig}`);
2085
+
2086
+ // Load and parse om.json using PersistableInMemoryMemory
2087
+ const storage = new PersistableInMemoryMemory({ readOnly: true });
2088
+ await storage.hydrate(omJsonPath);
2089
+
2090
+ // Get OM records
2091
+ const resourceId = `resource_${questionId}`;
2092
+ const records = await storage.getObservationalMemoryHistory(null, resourceId);
2093
+
2094
+ if (!records || records.length === 0) {
2095
+ console.log('āŒ No observational memory records found');
2096
+ return;
2097
+ }
2098
+
2099
+ const latestRecord = records[0];
2100
+ const activeObservations = latestRecord.activeObservations || '';
2101
+
2102
+ // Parse thread blocks from activeObservations
2103
+ const threadRegex = /<thread id="([^"]+)">([\s\S]*?)<\/thread>/g;
2104
+ const threadBlocks: { id: string; content: string; startIndex: number }[] = [];
2105
+ let match;
2106
+
2107
+ while ((match = threadRegex.exec(activeObservations)) !== null) {
2108
+ threadBlocks.push({
2109
+ id: match[1],
2110
+ content: match[2].trim(),
2111
+ startIndex: match.index,
2112
+ });
2113
+ }
2114
+
2115
+ console.log(`šŸ“Š Found ${threadBlocks.length} total thread blocks\n`);
2116
+
2117
+ // Count occurrences of each thread ID
2118
+ const threadCounts: Record<string, number> = {};
2119
+ for (const block of threadBlocks) {
2120
+ threadCounts[block.id] = (threadCounts[block.id] || 0) + 1;
2121
+ }
2122
+
2123
+ // Find duplicates
2124
+ const duplicates = Object.entries(threadCounts)
2125
+ .filter(([_, count]) => count > 1)
2126
+ .sort((a, b) => b[1] - a[1]);
2127
+
2128
+ const uniqueThreads = Object.keys(threadCounts).length;
2129
+
2130
+ console.log(`šŸ“ˆ Statistics:`);
2131
+ console.log(` Total thread blocks: ${threadBlocks.length}`);
2132
+ console.log(` Unique thread IDs: ${uniqueThreads}`);
2133
+ console.log(` Duplicate ratio: ${(threadBlocks.length / uniqueThreads).toFixed(2)}x\n`);
2134
+
2135
+ if (duplicates.length === 0) {
2136
+ console.log('āœ… No duplicate thread blocks found!');
2137
+ return;
2138
+ }
2139
+
2140
+ console.log(`āš ļø Found ${duplicates.length} thread IDs with duplicates:\n`);
2141
+
2142
+ for (const [threadId, count] of duplicates.slice(0, 10)) {
2143
+ console.log(` ${threadId}: ${count} occurrences`);
2144
+
2145
+ // Show the content of each duplicate to see if they're identical
2146
+ const blocks = threadBlocks.filter(b => b.id === threadId);
2147
+ const contents = blocks.map(b => b.content);
2148
+ const uniqueContents = new Set(contents);
2149
+
2150
+ if (uniqueContents.size === 1) {
2151
+ console.log(` └─ āš ļø All ${count} blocks have IDENTICAL content`);
2152
+ } else {
2153
+ console.log(` └─ ā„¹ļø ${uniqueContents.size} unique content variations (may be legitimate)`);
2154
+ }
2155
+ }
2156
+
2157
+ if (duplicates.length > 10) {
2158
+ console.log(`\n ... and ${duplicates.length - 10} more`);
2159
+ }
2160
+
2161
+ // Calculate token waste estimate
2162
+ const avgBlockSize = activeObservations.length / threadBlocks.length;
2163
+ const duplicateBlocks = threadBlocks.length - uniqueThreads;
2164
+ const estimatedWaste = Math.round(duplicateBlocks * avgBlockSize);
2165
+
2166
+ console.log(`\nšŸ“‰ Estimated token waste from duplicates: ~${estimatedWaste} characters`);
2167
+ }
2168
+
2169
+ // --------------------------------------------------------------------------
2170
+ // Baseline Check - Comprehensive Data Quality Check
2171
+ // --------------------------------------------------------------------------
2172
+
2173
+ private async runBaselineCheck(questionId: string): Promise<void> {
2174
+ console.log(`\nšŸ“Š BASELINE CHECK for ${questionId}\n`);
2175
+ console.log('═'.repeat(60));
2176
+
2177
+ // 1. Find the prepared data
2178
+ const configs = ['observational-memory', 'om-gpt5-mini', 'om-gpt5', 'om-gemini-3-pro', 'om-gemini-3-flash'];
2179
+ let omJsonPath: string | null = null;
2180
+ let foundConfig: string | null = null;
2181
+ let questionDir: string | null = null;
2182
+
2183
+ for (const config of configs) {
2184
+ const path = join(this.preparedDataDir, 'longmemeval_s', config, questionId, 'om.json');
2185
+ if (existsSync(path)) {
2186
+ omJsonPath = path;
2187
+ foundConfig = config;
2188
+ questionDir = join(this.preparedDataDir, 'longmemeval_s', config, questionId);
2189
+ break;
2190
+ }
2191
+ }
2192
+
2193
+ if (!omJsonPath || !questionDir) {
2194
+ console.log(`āŒ Could not find prepared data for question ${questionId}`);
2195
+ return;
2196
+ }
2197
+
2198
+ // 2. Get file modification time (preparation date)
2199
+ const { stat } = await import('fs/promises');
2200
+ const metaPath = join(questionDir, 'meta.json');
2201
+ let preparedDate = 'Unknown';
2202
+ if (existsSync(metaPath)) {
2203
+ const stats = await stat(metaPath);
2204
+ preparedDate = stats.mtime.toLocaleString();
2205
+ }
2206
+
2207
+ console.log(`\nšŸ“ DATA LOCATION`);
2208
+ console.log(` Config: ${foundConfig}`);
2209
+ console.log(` Path: ${questionDir}`);
2210
+ console.log(` Prepared: ${preparedDate}`);
2211
+
2212
+ // 3. Load storage and check for per-thread cursors (staleness)
2213
+ const storage = new PersistableInMemoryMemory({ readOnly: true });
2214
+ await storage.hydrate(omJsonPath);
2215
+
2216
+ const resourceId = `resource_${questionId}`;
2217
+
2218
+ // Get threads and check for cursors
2219
+ const threadsResult = await storage.listThreads({ filter: { resourceId }, perPage: false });
2220
+ const threads = threadsResult.threads || [];
2221
+ let threadsWithCursor = 0;
2222
+ let totalThreads = 0;
2223
+
2224
+ for (const thread of threads) {
2225
+ totalThreads++;
2226
+ const metadata = thread.metadata as Record<string, unknown> | undefined;
2227
+ const mastraOm = (metadata?.mastra as Record<string, unknown>)?.om as Record<string, unknown> | undefined;
2228
+ if (metadata?.lastObservedAt || mastraOm?.lastObservedAt) {
2229
+ threadsWithCursor++;
2230
+ }
2231
+ }
2232
+
2233
+ const isStale = totalThreads > 0 && threadsWithCursor === 0;
2234
+
2235
+ console.log(`\nšŸ• DATA FRESHNESS`);
2236
+ console.log(` Threads: ${totalThreads}`);
2237
+ console.log(` With per-thread cursors: ${threadsWithCursor}/${totalThreads}`);
2238
+ if (isStale) {
2239
+ console.log(` āš ļø STATUS: STALE (prepared before per-thread cursor fix)`);
2240
+ console.log(` → Recommend: pnpm prepare om -v full --question-id ${questionId} --force-regenerate -y`);
2241
+ } else {
2242
+ console.log(` āœ… STATUS: FRESH (has per-thread cursors)`);
2243
+ }
2244
+
2245
+ // 4. Check for duplicate observations
2246
+ const records = await storage.getObservationalMemoryHistory(null, resourceId);
2247
+ let duplicateRatio = 1.0;
2248
+ let totalBlocks = 0;
2249
+ let uniqueBlocks = 0;
2250
+
2251
+ if (records && records.length > 0) {
2252
+ const latestRecord = records[0];
2253
+ const activeObservations = latestRecord.activeObservations || '';
2254
+
2255
+ const threadRegex = /<thread id=\"([^\"]+)\">[\s\S]*?<\/thread>/g;
2256
+ const threadIds: string[] = [];
2257
+ let match;
2258
+ while ((match = threadRegex.exec(activeObservations)) !== null) {
2259
+ threadIds.push(match[1]);
2260
+ }
2261
+
2262
+ totalBlocks = threadIds.length;
2263
+ uniqueBlocks = new Set(threadIds).size;
2264
+ duplicateRatio = totalBlocks > 0 ? totalBlocks / uniqueBlocks : 1.0;
2265
+ }
2266
+
2267
+ console.log(`\nšŸ”„ DUPLICATE OBSERVATIONS`);
2268
+ console.log(` Total thread blocks: ${totalBlocks}`);
2269
+ console.log(` Unique thread IDs: ${uniqueBlocks}`);
2270
+ console.log(` Duplicate ratio: ${duplicateRatio.toFixed(2)}x`);
2271
+ if (duplicateRatio > 1.5) {
2272
+ console.log(` āš ļø HIGH DUPLICATION - may waste tokens`);
2273
+ console.log(` → Run: pnpm investigate --check-duplicates -q ${questionId}`);
2274
+ } else if (duplicateRatio > 1.0) {
2275
+ console.log(` ā„¹ļø Minor duplication (may be legitimate)`);
2276
+ } else {
2277
+ console.log(` āœ… No duplicates`);
2278
+ }
2279
+
2280
+ // 5. Get message and observation counts
2281
+ const messagesResult = await storage.listMessagesByResourceId({ resourceId, perPage: false });
2282
+ const messageCount = messagesResult.messages.length;
2283
+
2284
+ // Count observation lines
2285
+ let observationLineCount = 0;
2286
+ if (records && records.length > 0) {
2287
+ const activeObservations = records[0].activeObservations || '';
2288
+ observationLineCount = (activeObservations.match(/^\s*\*\s+/gm) || []).length;
2289
+ }
2290
+
2291
+ console.log(`\nšŸ“ˆ DATA VOLUME`);
2292
+ console.log(` Messages stored: ${messageCount}`);
2293
+ console.log(` Observation lines: ${observationLineCount}`);
2294
+ console.log(` OM records: ${records?.length || 0}`);
2295
+
2296
+ // 6. Summary
2297
+ console.log(`\n${'═'.repeat(60)}`);
2298
+ console.log(`šŸ“‹ SUMMARY`);
2299
+
2300
+ const issues: string[] = [];
2301
+ if (isStale) issues.push('STALE DATA');
2302
+ if (duplicateRatio > 1.5) issues.push('HIGH DUPLICATION');
2303
+
2304
+ if (issues.length === 0) {
2305
+ console.log(` āœ… No data quality issues detected`);
2306
+ } else {
2307
+ console.log(` āš ļø Issues found: ${issues.join(', ')}`);
2308
+ }
2309
+ console.log('');
2310
+ }
2311
+
2312
+ // --------------------------------------------------------------------------
2313
+ // Print Prepare Command for Stale Questions
2314
+ // --------------------------------------------------------------------------
2315
+
2316
+ private async printPrepareCommand(): Promise<void> {
2317
+ console.log('\nšŸ” Finding stale/partial questions in current investigation...\n');
2318
+
2319
+ // Get current investigation
2320
+ const investigations = await this.listInvestigations();
2321
+ if (investigations.length === 0) {
2322
+ console.log('āŒ No investigations found. Run: pnpm investigate <run-id>');
2323
+ return;
2324
+ }
2325
+
2326
+ // Use most recent investigation
2327
+ const currentInv = investigations[0];
2328
+ const progress = await this.loadProgress(currentInv);
2329
+ if (!progress) {
2330
+ console.log(`āŒ Could not load progress for ${currentInv}`);
2331
+ return;
2332
+ }
2333
+
2334
+ // Determine the base config to use for preparation
2335
+ let prepareConfig = progress.config;
2336
+ try {
2337
+ const { getMemoryConfig, isValidMemoryConfig } = await import('../config');
2338
+ if (isValidMemoryConfig(progress.config)) {
2339
+ const configDef = getMemoryConfig(progress.config);
2340
+ if (configDef?.readOnlyConfig && configDef?.baseConfig) {
2341
+ prepareConfig = configDef.baseConfig;
2342
+ }
2343
+ }
2344
+ } catch {
2345
+ // Use original config
2346
+ }
2347
+
2348
+ console.log(`šŸ“ Investigation: ${currentInv}`);
2349
+ console.log(` Config: ${progress.config}${prepareConfig !== progress.config ? ` (uses ${prepareConfig})` : ''}`);
2350
+ console.log(` Dataset: ${progress.dataset}\n`);
2351
+
2352
+ // Get pending questions
2353
+ const pendingQuestions = Object.entries(progress.questions)
2354
+ .filter(([_, q]) => q.status === 'pending')
2355
+ .map(([id]) => id);
2356
+
2357
+ console.log(`šŸ“‹ Pending questions: ${pendingQuestions.length}\n`);
2358
+
2359
+ // Check each pending question for staleness
2360
+ const staleQuestionIds: string[] = [];
2361
+
2362
+ for (const questionId of pendingQuestions) {
2363
+ const { omPath, config, dataset } = await this.findPreparedData(questionId);
2364
+ if (!omPath) continue;
2365
+
2366
+ const result = await this.checkSingleQuestion(questionId, omPath, config, dataset);
2367
+
2368
+ // Consider stale if:
2369
+ // 1. No threads have cursors (completely stale)
2370
+ // 2. Only some threads have cursors (partial)
2371
+ if (result.isStale || (result.threadCount > 0 && result.threadsWithCursor < result.threadCount)) {
2372
+ const status = result.threadsWithCursor === 0 ? 'STALE' : 'PARTIAL';
2373
+ console.log(` ${status}: ${questionId} (${result.reason})`);
2374
+ staleQuestionIds.push(questionId);
2375
+ }
2376
+ }
2377
+
2378
+ if (staleQuestionIds.length === 0) {
2379
+ console.log('āœ… No stale or partial questions found in pending queue!');
2380
+ return;
2381
+ }
2382
+
2383
+ console.log(`\nšŸ“‹ Found ${staleQuestionIds.length} stale/partial question(s)\n`);
2384
+ console.log('Run this command to re-prepare them:\n');
2385
+ console.log(
2386
+ `pnpm prepare ${prepareConfig} -v full --question-id ${staleQuestionIds.join(',')} --force-regenerate\n`,
2387
+ );
2388
+ }
2389
+
2390
+ // Deprecated - use printPrepareCommand instead
2391
+ private async prepareStaleQuestions(dryRun?: boolean): Promise<void> {
2392
+ console.log('\nāš ļø --prepare-stale is deprecated. Use --print-prepare-command instead.\n');
2393
+ await this.printPrepareCommand();
2394
+ }
2395
+
2396
+ private showHelp(): void {
2397
+ console.log(`
2398
+ šŸ“‹ Investigation Workflow
2399
+
2400
+ Usage:
2401
+ pnpm investigate <run-id> Setup investigation from a benchmark run
2402
+ pnpm investigate --status Show progress across all investigations
2403
+ pnpm investigate --next Open next uninvestigated question
2404
+ pnpm investigate --done <question-id> Mark a question as investigated
2405
+ pnpm investigate --sync Sync fixes to dataset
2406
+
2407
+ Investigation Utilities:
2408
+ pnpm investigate --inspect <question-id> Inspect question data
2409
+ pnpm investigate --search <keyword> -q <id> Search observations
2410
+ pnpm investigate --trace <keyword> -q <id> Trace info through pipeline
2411
+
2412
+ Data Quality:
2413
+ pnpm investigate --baseline -q <id> Comprehensive data quality check
2414
+ pnpm investigate --check-stale Check all data for staleness
2415
+ pnpm investigate --check-stale -q <id> Check specific question
2416
+ pnpm investigate --check-stale --stale-only List only stale questions
2417
+ pnpm investigate --check-duplicates -q <id> Check for duplicate thread blocks
2418
+
2419
+ Dataset Improvements:
2420
+ pnpm investigate --improve <id> --improve-question "..." --improve-answer "..." --improve-note "..."
2421
+
2422
+ Examples:
2423
+ pnpm investigate run_1234567890
2424
+ pnpm investigate --inspect 07741c45
2425
+ pnpm investigate --search "sneaker" -q 07741c45
2426
+ pnpm investigate --trace "shoe rack" -q 07741c45
2427
+ pnpm investigate --check-stale --stale-only Find questions to re-prepare
2428
+
2429
+ Options:
2430
+ --output <dir> Investigation output directory (default: ./investigations)
2431
+ --results <dir> Results directory (default: ./results)
2432
+ --prepared-data <dir> Prepared data directory (default: ./prepared-data)
2433
+ --dataset <dir> Dataset directory (default: ./data)
2434
+ --editor <cmd> Editor command (default: $EDITOR or 'code')
2435
+ `);
2436
+ }
2437
+ }