@mastra/longmemeval 0.0.0-error-handler-fix-20251020202607 ā 0.0.0-execa-dynamic-import-20260304221256
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +847 -6
- package/LICENSE.md +15 -0
- package/docs/INVESTIGATION_WORKFLOW.md +314 -0
- package/package.json +34 -26
- package/scripts/download.ts +22 -1
- package/scripts/generate-wm-templates.ts +2 -1
- package/src/cli.ts +967 -155
- package/src/commands/__tests__/prepare.test.ts +11 -6
- package/src/commands/__tests__/run.test.ts +10 -8
- package/src/commands/clean.ts +147 -0
- package/src/commands/deterministic-ids.ts +126 -0
- package/src/commands/find-prohibited.ts +299 -0
- package/src/commands/investigate.ts +2437 -0
- package/src/commands/list-partial.ts +137 -0
- package/src/commands/obscure-thread-ids.ts +237 -0
- package/src/commands/partial-results.ts +198 -0
- package/src/commands/precompute-embeddings.ts +302 -0
- package/src/commands/prepare.ts +732 -105
- package/src/commands/reconcile.ts +383 -0
- package/src/commands/run.ts +1240 -121
- package/src/commands/sessions.ts +126 -0
- package/src/commands/sync.ts +152 -0
- package/src/commands/test-cleaned.ts +53 -0
- package/src/commands/test-incremental.ts +120 -0
- package/src/commands/test-messages.ts +95 -0
- package/src/commands/test-prohibited.ts +244 -0
- package/src/commands/test-thread.ts +136 -0
- package/src/commands/tokens.ts +625 -0
- package/src/config.ts +1253 -72
- package/src/data/types.ts +48 -6
- package/src/embeddings/cached-openai-embedding-model.ts +0 -1
- package/src/evaluation/longmemeval-metric.ts +186 -103
- package/src/processors/content-sanitizer.ts +153 -0
- package/src/processors/date-injector.ts +111 -0
- package/src/processors/index.ts +3 -0
- package/src/processors/observation-semantic-filter.ts +1142 -0
- package/src/retry-model.ts +5 -3
- package/src/storage/__tests__/benchmark-store.test.ts +18 -35
- package/src/storage/benchmark-store.ts +140 -153
- package/src/storage/benchmark-vector.ts +1 -1
- package/src/storage/index.ts +1 -0
- package/src/storage/persistable-inmemory.ts +133 -0
- package/src/test-utils/mock-model.ts +35 -0
|
@@ -0,0 +1,2437 @@
|
|
|
1
|
+
import { existsSync } from 'fs';
|
|
2
|
+
import { readFile, writeFile, mkdir, readdir } from 'fs/promises';
|
|
3
|
+
import { join, dirname } from 'path';
|
|
4
|
+
import { DatasetLoader } from '../data/loader';
|
|
5
|
+
import { PersistableInMemoryMemory } from '../storage';
|
|
6
|
+
import type { LongMemEvalQuestion, FailureCategory } from '../data/types';
|
|
7
|
+
|
|
8
|
+
// Re-export FailureCategory for backwards compatibility
|
|
9
|
+
export type { FailureCategory } from '../data/types';
|
|
10
|
+
|
|
11
|
+
export interface QuestionProgress {
|
|
12
|
+
status: 'pending' | 'investigated' | 'fix-implemented' | 'synced';
|
|
13
|
+
category?: FailureCategory;
|
|
14
|
+
investigatedAt?: string;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface InvestigationProgress {
|
|
18
|
+
runId: string;
|
|
19
|
+
config: string;
|
|
20
|
+
dataset: string;
|
|
21
|
+
createdAt: string;
|
|
22
|
+
totalFailed: number;
|
|
23
|
+
investigated: number;
|
|
24
|
+
questions: Record<string, QuestionProgress>;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface InvestigateOptions {
|
|
28
|
+
runId?: string;
|
|
29
|
+
list?: boolean;
|
|
30
|
+
config?: string;
|
|
31
|
+
status?: boolean;
|
|
32
|
+
next?: boolean;
|
|
33
|
+
done?: string;
|
|
34
|
+
sync?: boolean;
|
|
35
|
+
fixed?: string; // Mark a question as fix-implemented
|
|
36
|
+
outputDir?: string;
|
|
37
|
+
resultsDir?: string;
|
|
38
|
+
preparedDataDir?: string;
|
|
39
|
+
datasetDir?: string;
|
|
40
|
+
editor?: string;
|
|
41
|
+
// Investigation utilities
|
|
42
|
+
search?: string; // Search observations for keywords
|
|
43
|
+
trace?: string; // Trace information flow for a keyword
|
|
44
|
+
questionId?: string; // Question ID for search/trace/date/session
|
|
45
|
+
inspect?: string; // Inspect a specific question's data
|
|
46
|
+
date?: string; // View observations around a specific date (e.g., "2023/05/29" or "May 29")
|
|
47
|
+
context?: number; // Number of days of context around the date (default: 1)
|
|
48
|
+
session?: number; // View a specific session from original dataset
|
|
49
|
+
listSessions?: boolean; // List all sessions with dates for a question
|
|
50
|
+
// Data freshness detection
|
|
51
|
+
checkStale?: boolean; // Check if prepared data is stale (pre-cursor-fix)
|
|
52
|
+
staleOnly?: boolean; // Only list stale questions from failures
|
|
53
|
+
// Original dataset search
|
|
54
|
+
searchOriginal?: string; // Search original dataset for a keyword
|
|
55
|
+
// Improve question/answer in dataset
|
|
56
|
+
improve?: string; // Question ID to add improvements for
|
|
57
|
+
improveQuestion?: string; // Improved question text
|
|
58
|
+
improveAnswer?: string; // Improved answer text
|
|
59
|
+
improveNote?: string; // Improvement note
|
|
60
|
+
category?: FailureCategory; // Failure category for this question
|
|
61
|
+
clearImproved?: string | boolean; // Clear specific field(s): 'all', 'question', 'answer', 'note', 'category' or true for all
|
|
62
|
+
// Check for duplicate observations
|
|
63
|
+
checkDuplicates?: boolean; // Check for duplicate thread blocks in observations
|
|
64
|
+
// Baseline check - comprehensive data quality check
|
|
65
|
+
baseline?: boolean; // Run all data quality checks for a question
|
|
66
|
+
// Prepare stale questions
|
|
67
|
+
prepareStale?: boolean; // Find and prepare all stale pending questions (deprecated, use --print-prepare-command)
|
|
68
|
+
dryRun?: boolean; // Just show what would be prepared, don't run
|
|
69
|
+
printPrepareCommand?: boolean; // Print a prepare command for stale questions
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
interface FailuresFile {
|
|
73
|
+
questionIds: string[];
|
|
74
|
+
runId: string;
|
|
75
|
+
config: string;
|
|
76
|
+
dataset: string;
|
|
77
|
+
timestamp: string;
|
|
78
|
+
totalFailed: number;
|
|
79
|
+
totalQuestions: number;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
interface EvaluationResult {
|
|
83
|
+
question_id: string;
|
|
84
|
+
question: string;
|
|
85
|
+
expected_answer: string;
|
|
86
|
+
hypothesis: string; // The agent's answer
|
|
87
|
+
is_correct: boolean;
|
|
88
|
+
question_type: string;
|
|
89
|
+
improved_question?: string;
|
|
90
|
+
improved_answer?: string;
|
|
91
|
+
improved_is_correct?: boolean;
|
|
92
|
+
has_improvement_info?: boolean;
|
|
93
|
+
improved_regression?: boolean;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// ============================================================================
|
|
97
|
+
// Analysis Template
|
|
98
|
+
// ============================================================================
|
|
99
|
+
|
|
100
|
+
function generateAnalysisTemplate(
|
|
101
|
+
questionId: string,
|
|
102
|
+
question: string,
|
|
103
|
+
expectedAnswer: string,
|
|
104
|
+
actualAnswer: string,
|
|
105
|
+
questionType: string,
|
|
106
|
+
improvedQuestion?: string,
|
|
107
|
+
improvedAnswer?: string,
|
|
108
|
+
): string {
|
|
109
|
+
return `# Investigation: ${questionId}
|
|
110
|
+
|
|
111
|
+
## Question Type
|
|
112
|
+
${questionType}
|
|
113
|
+
|
|
114
|
+
## Question
|
|
115
|
+
${question}
|
|
116
|
+
|
|
117
|
+
## Expected Answer
|
|
118
|
+
${expectedAnswer}
|
|
119
|
+
|
|
120
|
+
## Agent Answer
|
|
121
|
+
${actualAnswer}
|
|
122
|
+
|
|
123
|
+
${improvedQuestion ? `## Improved Question (existing)\n${improvedQuestion}\n` : ''}
|
|
124
|
+
${improvedAnswer ? `## Improved Answer (existing)\n${improvedAnswer}\n` : ''}
|
|
125
|
+
|
|
126
|
+
---
|
|
127
|
+
|
|
128
|
+
## Failure Category
|
|
129
|
+
- [ ] Observer missed critical information
|
|
130
|
+
- [ ] Reflector lost/merged information incorrectly
|
|
131
|
+
- [ ] Agent reasoning error (had info, wrong conclusion)
|
|
132
|
+
- [ ] Ambiguous/poorly-worded question
|
|
133
|
+
- [ ] Dataset inconsistency/error
|
|
134
|
+
- [ ] RAG retrieval miss (if applicable)
|
|
135
|
+
- [ ] Other: ___
|
|
136
|
+
|
|
137
|
+
## Root Cause Analysis
|
|
138
|
+
<!-- What specifically went wrong? -->
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
## Evidence
|
|
142
|
+
<!-- Quote relevant parts of om.md, original data, etc. -->
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
---
|
|
146
|
+
|
|
147
|
+
## Potential Improvements
|
|
148
|
+
|
|
149
|
+
### Observer/Reflector Changes
|
|
150
|
+
- **Likelihood**: Low / Medium / High
|
|
151
|
+
- **Reasoning**:
|
|
152
|
+
- **Suggested prompt change**:
|
|
153
|
+
|
|
154
|
+
### Fixed Question/Answer
|
|
155
|
+
- **Likelihood**: Low / Medium / High
|
|
156
|
+
- **improved_question**:
|
|
157
|
+
- **improved_answer**:
|
|
158
|
+
- **improvement_note**:
|
|
159
|
+
|
|
160
|
+
### Other Improvements
|
|
161
|
+
<!-- Prompt changes, architectural changes, etc. -->
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
---
|
|
165
|
+
|
|
166
|
+
## Status
|
|
167
|
+
- [ ] Investigated
|
|
168
|
+
- [ ] Fix implemented
|
|
169
|
+
- [ ] Synced to longmemeval_s.json
|
|
170
|
+
`;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// ============================================================================
|
|
174
|
+
// Investigation Command
|
|
175
|
+
// ============================================================================
|
|
176
|
+
|
|
177
|
+
export class InvestigateCommand {
|
|
178
|
+
private investigationsDir: string;
|
|
179
|
+
private resultsDir: string;
|
|
180
|
+
private preparedDataDir: string;
|
|
181
|
+
private datasetDir: string;
|
|
182
|
+
private editor: string;
|
|
183
|
+
|
|
184
|
+
constructor(options: InvestigateOptions = {}) {
|
|
185
|
+
this.investigationsDir = options.outputDir || './investigations';
|
|
186
|
+
this.resultsDir = options.resultsDir || './results';
|
|
187
|
+
this.preparedDataDir = options.preparedDataDir || './prepared-data';
|
|
188
|
+
this.datasetDir = options.datasetDir || './data';
|
|
189
|
+
this.editor = options.editor || process.env.EDITOR || 'code';
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
async run(options: InvestigateOptions): Promise<void> {
|
|
193
|
+
// Handle different modes
|
|
194
|
+
if (options.list) {
|
|
195
|
+
await this.listFailures(options.config);
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
if (options.status) {
|
|
200
|
+
await this.showStatus();
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
if (options.next) {
|
|
205
|
+
await this.openNext();
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
if (options.done) {
|
|
210
|
+
await this.markDone(options.done);
|
|
211
|
+
return;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
if (options.fixed) {
|
|
215
|
+
await this.markFixed(options.fixed);
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
if (options.sync) {
|
|
220
|
+
await this.syncToDataset();
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// Investigation utilities
|
|
225
|
+
if (options.inspect) {
|
|
226
|
+
await this.inspectQuestion(options.inspect);
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
if (options.search && options.questionId) {
|
|
231
|
+
await this.searchObservations(options.questionId, options.search);
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
if (options.trace && options.questionId) {
|
|
236
|
+
await this.traceInformation(options.questionId, options.trace);
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// Date-based observation viewer
|
|
241
|
+
if (options.date && options.questionId) {
|
|
242
|
+
await this.viewObservationsAroundDate(options.questionId, options.date, options.context ?? 1);
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// Session viewer
|
|
247
|
+
if (options.listSessions && options.questionId) {
|
|
248
|
+
await this.listSessions(options.questionId);
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
if (options.session !== undefined && options.questionId) {
|
|
253
|
+
await this.viewSession(options.questionId, options.session);
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// Data freshness detection
|
|
258
|
+
if (options.checkStale) {
|
|
259
|
+
await this.checkStaleData(options.questionId, options.staleOnly);
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
// Original dataset search
|
|
264
|
+
if (options.searchOriginal && options.questionId) {
|
|
265
|
+
await this.searchOriginalDataset(options.questionId, options.searchOriginal);
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
// Improve question/answer in dataset
|
|
270
|
+
if (options.improve) {
|
|
271
|
+
await this.addImprovement(
|
|
272
|
+
options.improve,
|
|
273
|
+
options.improveQuestion,
|
|
274
|
+
options.improveAnswer,
|
|
275
|
+
options.improveNote,
|
|
276
|
+
options.category,
|
|
277
|
+
options.clearImproved,
|
|
278
|
+
);
|
|
279
|
+
return;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// Check for duplicate observations
|
|
283
|
+
if (options.checkDuplicates) {
|
|
284
|
+
await this.checkDuplicateObservations(options.questionId);
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
// Baseline check - comprehensive data quality check
|
|
289
|
+
if (options.baseline && options.questionId) {
|
|
290
|
+
await this.runBaselineCheck(options.questionId);
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
// Prepare stale questions (deprecated)
|
|
295
|
+
if (options.prepareStale) {
|
|
296
|
+
await this.prepareStaleQuestions(options.dryRun);
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
// Print prepare command for stale questions
|
|
301
|
+
if (options.printPrepareCommand) {
|
|
302
|
+
await this.printPrepareCommand();
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
// Default: setup investigation from run
|
|
307
|
+
if (options.runId) {
|
|
308
|
+
await this.setupInvestigation(options.runId);
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
// No args: show help
|
|
313
|
+
this.showHelp();
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
// --------------------------------------------------------------------------
|
|
317
|
+
// List Failures
|
|
318
|
+
// --------------------------------------------------------------------------
|
|
319
|
+
|
|
320
|
+
private async listFailures(configFilter?: string): Promise<void> {
|
|
321
|
+
console.log('\nš Scanning for benchmark runs with failures...\n');
|
|
322
|
+
|
|
323
|
+
interface RunInfo {
|
|
324
|
+
runId: string;
|
|
325
|
+
config: string;
|
|
326
|
+
dataset: string;
|
|
327
|
+
timestamp: string;
|
|
328
|
+
totalFailed: number;
|
|
329
|
+
totalQuestions: number;
|
|
330
|
+
failuresPath: string;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
const runs: RunInfo[] = [];
|
|
334
|
+
|
|
335
|
+
// Scan results directory for all configs
|
|
336
|
+
if (!existsSync(this.resultsDir)) {
|
|
337
|
+
console.log('No results directory found.');
|
|
338
|
+
return;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
const configs = await readdir(this.resultsDir);
|
|
342
|
+
|
|
343
|
+
for (const config of configs) {
|
|
344
|
+
// Skip if filtering by config and doesn't match
|
|
345
|
+
if (configFilter && !config.includes(configFilter)) {
|
|
346
|
+
continue;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
const configDir = join(this.resultsDir, config);
|
|
350
|
+
const stat = await import('fs/promises').then(fs => fs.stat(configDir));
|
|
351
|
+
if (!stat.isDirectory()) continue;
|
|
352
|
+
|
|
353
|
+
const runDirs = await readdir(configDir);
|
|
354
|
+
|
|
355
|
+
for (const runDir of runDirs) {
|
|
356
|
+
if (!runDir.startsWith('run_')) continue;
|
|
357
|
+
|
|
358
|
+
const failuresPath = join(configDir, runDir, 'failures.json');
|
|
359
|
+
if (!existsSync(failuresPath)) continue;
|
|
360
|
+
|
|
361
|
+
try {
|
|
362
|
+
const failures = JSON.parse(await readFile(failuresPath, 'utf-8')) as FailuresFile;
|
|
363
|
+
runs.push({
|
|
364
|
+
runId: failures.runId,
|
|
365
|
+
config: failures.config,
|
|
366
|
+
dataset: failures.dataset,
|
|
367
|
+
timestamp: failures.timestamp,
|
|
368
|
+
totalFailed: failures.totalFailed,
|
|
369
|
+
totalQuestions: failures.totalQuestions,
|
|
370
|
+
failuresPath,
|
|
371
|
+
});
|
|
372
|
+
} catch {
|
|
373
|
+
// Skip invalid files
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
if (runs.length === 0) {
|
|
379
|
+
console.log('No runs with failures found.');
|
|
380
|
+
return;
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
// Sort by timestamp (newest first)
|
|
384
|
+
runs.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
|
|
385
|
+
|
|
386
|
+
// Group by config
|
|
387
|
+
const byConfig = new Map<string, RunInfo[]>();
|
|
388
|
+
for (const run of runs) {
|
|
389
|
+
const key = `${run.dataset}/${run.config}`;
|
|
390
|
+
if (!byConfig.has(key)) {
|
|
391
|
+
byConfig.set(key, []);
|
|
392
|
+
}
|
|
393
|
+
byConfig.get(key)!.push(run);
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
// Display
|
|
397
|
+
console.log(`Found ${runs.length} runs with failures across ${byConfig.size} configs:\n`);
|
|
398
|
+
|
|
399
|
+
for (const [configKey, configRuns] of byConfig) {
|
|
400
|
+
console.log(`š ${configKey}`);
|
|
401
|
+
|
|
402
|
+
// Show latest run prominently
|
|
403
|
+
const latest = configRuns[0];
|
|
404
|
+
const failRate = ((latest.totalFailed / latest.totalQuestions) * 100).toFixed(1);
|
|
405
|
+
console.log(` Latest: ${latest.runId}`);
|
|
406
|
+
console.log(` Failed: ${latest.totalFailed}/${latest.totalQuestions} (${failRate}%)`);
|
|
407
|
+
console.log(` Date: ${new Date(latest.timestamp).toLocaleString()}`);
|
|
408
|
+
|
|
409
|
+
if (configRuns.length > 1) {
|
|
410
|
+
console.log(` (${configRuns.length - 1} older runs)`);
|
|
411
|
+
}
|
|
412
|
+
console.log();
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
// Show usage hint
|
|
416
|
+
console.log('To investigate a run:');
|
|
417
|
+
console.log(` pnpm investigate <run-id>`);
|
|
418
|
+
console.log(` pnpm investigate ${runs[0].runId}`);
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
// --------------------------------------------------------------------------
|
|
422
|
+
// Setup Investigation
|
|
423
|
+
// --------------------------------------------------------------------------
|
|
424
|
+
|
|
425
|
+
private async setupInvestigation(runIdOrPath: string): Promise<void> {
|
|
426
|
+
console.log(`\nš Setting up investigation for: ${runIdOrPath}\n`);
|
|
427
|
+
|
|
428
|
+
// Find the failures.json file
|
|
429
|
+
const { failuresPath, failures } = await this.findFailures(runIdOrPath);
|
|
430
|
+
console.log(`š Found failures file: ${failuresPath}`);
|
|
431
|
+
console.log(` Config: ${failures.config}`);
|
|
432
|
+
console.log(` Dataset: ${failures.dataset}`);
|
|
433
|
+
console.log(` Failed: ${failures.totalFailed}/${failures.totalQuestions}`);
|
|
434
|
+
|
|
435
|
+
// Create investigation directory
|
|
436
|
+
const investigationDir = join(this.investigationsDir, failures.runId);
|
|
437
|
+
await mkdir(investigationDir, { recursive: true });
|
|
438
|
+
|
|
439
|
+
// Load existing progress or create new
|
|
440
|
+
const progressPath = join(investigationDir, 'progress.json');
|
|
441
|
+
let progress: InvestigationProgress;
|
|
442
|
+
|
|
443
|
+
if (existsSync(progressPath)) {
|
|
444
|
+
progress = JSON.parse(await readFile(progressPath, 'utf-8'));
|
|
445
|
+
console.log(`\nš Resuming existing investigation (${progress.investigated}/${progress.totalFailed} done)`);
|
|
446
|
+
} else {
|
|
447
|
+
progress = {
|
|
448
|
+
runId: failures.runId,
|
|
449
|
+
config: failures.config,
|
|
450
|
+
dataset: failures.dataset,
|
|
451
|
+
createdAt: new Date().toISOString(),
|
|
452
|
+
totalFailed: failures.totalFailed,
|
|
453
|
+
investigated: 0,
|
|
454
|
+
questions: {},
|
|
455
|
+
};
|
|
456
|
+
|
|
457
|
+
for (const qid of failures.questionIds) {
|
|
458
|
+
progress.questions[qid] = { status: 'pending' };
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
// Load results.jsonl to get actual answers
|
|
463
|
+
const resultsDir = dirname(failuresPath);
|
|
464
|
+
const resultsPath = join(resultsDir, 'results.jsonl');
|
|
465
|
+
const resultsMap = new Map<string, EvaluationResult>();
|
|
466
|
+
|
|
467
|
+
if (existsSync(resultsPath)) {
|
|
468
|
+
const resultsContent = await readFile(resultsPath, 'utf-8');
|
|
469
|
+
for (const line of resultsContent.split('\n').filter(l => l.trim())) {
|
|
470
|
+
try {
|
|
471
|
+
const result = JSON.parse(line) as EvaluationResult;
|
|
472
|
+
resultsMap.set(result.question_id, result);
|
|
473
|
+
} catch {
|
|
474
|
+
// Skip invalid lines
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
// Load dataset for original questions
|
|
480
|
+
const loader = new DatasetLoader(this.datasetDir);
|
|
481
|
+
const dataset = await loader.loadDataset(failures.dataset as 'longmemeval_s' | 'longmemeval_m');
|
|
482
|
+
const questionMap = new Map<string, LongMemEvalQuestion>();
|
|
483
|
+
for (const q of dataset) {
|
|
484
|
+
questionMap.set(q.question_id, q);
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
// Setup each failed question
|
|
488
|
+
let created = 0;
|
|
489
|
+
let skipped = 0;
|
|
490
|
+
|
|
491
|
+
for (const questionId of failures.questionIds) {
|
|
492
|
+
const questionDir = join(investigationDir, questionId);
|
|
493
|
+
const dataDir = join(questionDir, 'data');
|
|
494
|
+
const analysisPath = join(questionDir, 'analysis.md');
|
|
495
|
+
|
|
496
|
+
// Skip if already exists
|
|
497
|
+
if (existsSync(analysisPath)) {
|
|
498
|
+
skipped++;
|
|
499
|
+
continue;
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
await mkdir(dataDir, { recursive: true });
|
|
503
|
+
|
|
504
|
+
// Get question data
|
|
505
|
+
const question = questionMap.get(questionId);
|
|
506
|
+
const result = resultsMap.get(questionId);
|
|
507
|
+
|
|
508
|
+
if (!question) {
|
|
509
|
+
console.warn(` ā ļø Question ${questionId} not found in dataset`);
|
|
510
|
+
continue;
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
// Copy original question data
|
|
514
|
+
await writeFile(join(dataDir, 'original.json'), JSON.stringify(question, null, 2));
|
|
515
|
+
|
|
516
|
+
// Copy result if available
|
|
517
|
+
if (result) {
|
|
518
|
+
await writeFile(join(dataDir, 'result.json'), JSON.stringify(result, null, 2));
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
// Copy prepared data files
|
|
522
|
+
const preparedDir = join(this.preparedDataDir, failures.dataset, failures.config, questionId);
|
|
523
|
+
|
|
524
|
+
const filesToCopy = ['om.md', 'om.json', 'meta.json'];
|
|
525
|
+
for (const file of filesToCopy) {
|
|
526
|
+
const srcPath = join(preparedDir, file);
|
|
527
|
+
if (existsSync(srcPath)) {
|
|
528
|
+
const content = await readFile(srcPath, 'utf-8');
|
|
529
|
+
await writeFile(join(dataDir, file), content);
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
// Generate analysis template
|
|
534
|
+
const template = generateAnalysisTemplate(
|
|
535
|
+
questionId,
|
|
536
|
+
question.question,
|
|
537
|
+
question.answer,
|
|
538
|
+
result?.hypothesis || '(not available)',
|
|
539
|
+
question.question_type,
|
|
540
|
+
question.improved_question,
|
|
541
|
+
question.improved_answer,
|
|
542
|
+
);
|
|
543
|
+
|
|
544
|
+
await writeFile(analysisPath, template);
|
|
545
|
+
created++;
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
// Save progress
|
|
549
|
+
await writeFile(progressPath, JSON.stringify(progress, null, 2));
|
|
550
|
+
|
|
551
|
+
console.log(`\nā
Investigation setup complete!`);
|
|
552
|
+
console.log(` Created: ${created} new question directories`);
|
|
553
|
+
console.log(` Skipped: ${skipped} existing directories`);
|
|
554
|
+
console.log(` Location: ${investigationDir}`);
|
|
555
|
+
console.log(`\nš Next steps:`);
|
|
556
|
+
console.log(` pnpm investigate --status # Check progress`);
|
|
557
|
+
console.log(` pnpm investigate --next # Open next question`);
|
|
558
|
+
console.log(` pnpm investigate --done <question-id> # Mark as investigated`);
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
// --------------------------------------------------------------------------
|
|
562
|
+
// Status
|
|
563
|
+
// --------------------------------------------------------------------------
|
|
564
|
+
|
|
565
|
+
private async showStatus(): Promise<void> {
|
|
566
|
+
const investigations = await this.listInvestigations();
|
|
567
|
+
|
|
568
|
+
if (investigations.length === 0) {
|
|
569
|
+
console.log('\nš No investigations found.');
|
|
570
|
+
console.log(' Run: pnpm investigate <run-id> to start one.\n');
|
|
571
|
+
return;
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
console.log('\nš Investigation Status\n');
|
|
575
|
+
|
|
576
|
+
for (const inv of investigations) {
|
|
577
|
+
const progress = await this.loadProgress(inv);
|
|
578
|
+
if (!progress) continue;
|
|
579
|
+
|
|
580
|
+
const pending = Object.values(progress.questions).filter(q => q.status === 'pending').length;
|
|
581
|
+
const investigated = Object.values(progress.questions).filter(q => q.status === 'investigated').length;
|
|
582
|
+
const fixImplemented = Object.values(progress.questions).filter(q => q.status === 'fix-implemented').length;
|
|
583
|
+
const synced = Object.values(progress.questions).filter(q => q.status === 'synced').length;
|
|
584
|
+
|
|
585
|
+
const pct = (((progress.totalFailed - pending) / progress.totalFailed) * 100).toFixed(1);
|
|
586
|
+
|
|
587
|
+
console.log(`š ${inv}`);
|
|
588
|
+
console.log(` Config: ${progress.config}`);
|
|
589
|
+
console.log(` Progress: ${progress.totalFailed - pending}/${progress.totalFailed} (${pct}%)`);
|
|
590
|
+
console.log(` āā Pending: ${pending}`);
|
|
591
|
+
console.log(` āā Investigated: ${investigated}`);
|
|
592
|
+
console.log(` āā Fix Implemented: ${fixImplemented}`);
|
|
593
|
+
console.log(` āā Synced: ${synced}`);
|
|
594
|
+
|
|
595
|
+
// Show category breakdown if any investigated
|
|
596
|
+
if (investigated + fixImplemented + synced > 0) {
|
|
597
|
+
const categories = new Map<string, number>();
|
|
598
|
+
for (const q of Object.values(progress.questions)) {
|
|
599
|
+
if (q.category) {
|
|
600
|
+
categories.set(q.category, (categories.get(q.category) || 0) + 1);
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
if (categories.size > 0) {
|
|
605
|
+
console.log(` Categories:`);
|
|
606
|
+
for (const [cat, count] of [...categories.entries()].sort((a, b) => b[1] - a[1])) {
|
|
607
|
+
console.log(` ${cat}: ${count}`);
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
console.log('');
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
// --------------------------------------------------------------------------
|
|
617
|
+
// Open Next
|
|
618
|
+
// --------------------------------------------------------------------------
|
|
619
|
+
|
|
620
|
+
private async openNext(): Promise<void> {
|
|
621
|
+
const investigations = await this.listInvestigations();
|
|
622
|
+
|
|
623
|
+
if (investigations.length === 0) {
|
|
624
|
+
console.log('\nš No investigations found.\n');
|
|
625
|
+
return;
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
// Use most recent investigation
|
|
629
|
+
const latestInv = investigations[investigations.length - 1];
|
|
630
|
+
const progress = await this.loadProgress(latestInv);
|
|
631
|
+
|
|
632
|
+
if (!progress) {
|
|
633
|
+
console.log('\nā Could not load progress.\n');
|
|
634
|
+
return;
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
// Find next pending question
|
|
638
|
+
const pendingId = Object.entries(progress.questions).find(([_, q]) => q.status === 'pending')?.[0];
|
|
639
|
+
|
|
640
|
+
if (!pendingId) {
|
|
641
|
+
console.log('\nš All questions investigated!\n');
|
|
642
|
+
return;
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
const analysisPath = join(this.investigationsDir, latestInv, pendingId, 'analysis.md');
|
|
646
|
+
|
|
647
|
+
if (!existsSync(analysisPath)) {
|
|
648
|
+
console.log(`\nā Analysis file not found: ${analysisPath}\n`);
|
|
649
|
+
return;
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
console.log(`\nš Opening: ${pendingId}`);
|
|
653
|
+
console.log(` File: ${analysisPath}`);
|
|
654
|
+
console.log(` Editor: ${this.editor}\n`);
|
|
655
|
+
|
|
656
|
+
// Open in editor
|
|
657
|
+
const { spawn } = await import('child_process');
|
|
658
|
+
spawn(this.editor, [analysisPath], {
|
|
659
|
+
detached: true,
|
|
660
|
+
stdio: 'ignore',
|
|
661
|
+
}).unref();
|
|
662
|
+
|
|
663
|
+
// Also print some context
|
|
664
|
+
const dataDir = join(this.investigationsDir, latestInv, pendingId, 'data');
|
|
665
|
+
const resultPath = join(dataDir, 'result.json');
|
|
666
|
+
|
|
667
|
+
if (existsSync(resultPath)) {
|
|
668
|
+
const result = JSON.parse(await readFile(resultPath, 'utf-8')) as EvaluationResult;
|
|
669
|
+
const expectedStr = String(result.expected_answer);
|
|
670
|
+
const hypothesisStr = String(result.hypothesis);
|
|
671
|
+
console.log(`š Quick Context:`);
|
|
672
|
+
console.log(` Type: ${result.question_type}`);
|
|
673
|
+
console.log(` Q: ${result.question.substring(0, 100)}...`);
|
|
674
|
+
console.log(` Expected: ${expectedStr.substring(0, 100)}${expectedStr.length > 100 ? '...' : ''}`);
|
|
675
|
+
console.log(` Got: ${hypothesisStr.substring(0, 100)}${hypothesisStr.length > 100 ? '...' : ''}`);
|
|
676
|
+
console.log('');
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
console.log(`When done, run: pnpm investigate --done ${pendingId}\n`);
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
// --------------------------------------------------------------------------
|
|
683
|
+
// Mark Done
|
|
684
|
+
// --------------------------------------------------------------------------
|
|
685
|
+
|
|
686
|
+
private async markDone(questionId: string): Promise<void> {
|
|
687
|
+
const investigations = await this.listInvestigations();
|
|
688
|
+
|
|
689
|
+
if (investigations.length === 0) {
|
|
690
|
+
console.log('\nš No investigations found.\n');
|
|
691
|
+
return;
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
// Find investigation containing this question
|
|
695
|
+
let foundInv: string | null = null;
|
|
696
|
+
let progress: InvestigationProgress | null = null;
|
|
697
|
+
|
|
698
|
+
for (const inv of investigations) {
|
|
699
|
+
const p = await this.loadProgress(inv);
|
|
700
|
+
if (p && questionId in p.questions) {
|
|
701
|
+
foundInv = inv;
|
|
702
|
+
progress = p;
|
|
703
|
+
break;
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
if (!foundInv || !progress) {
|
|
708
|
+
console.log(`\nā Question ${questionId} not found in any investigation.\n`);
|
|
709
|
+
return;
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
// Parse analysis.md to extract category
|
|
713
|
+
const analysisPath = join(this.investigationsDir, foundInv, questionId, 'analysis.md');
|
|
714
|
+
let category: FailureCategory | undefined;
|
|
715
|
+
|
|
716
|
+
if (existsSync(analysisPath)) {
|
|
717
|
+
const content = await readFile(analysisPath, 'utf-8');
|
|
718
|
+
category = this.extractCategory(content);
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
// Update progress
|
|
722
|
+
progress.questions[questionId] = {
|
|
723
|
+
status: 'investigated',
|
|
724
|
+
category,
|
|
725
|
+
investigatedAt: new Date().toISOString(),
|
|
726
|
+
};
|
|
727
|
+
|
|
728
|
+
progress.investigated = Object.values(progress.questions).filter(q => q.status !== 'pending').length;
|
|
729
|
+
|
|
730
|
+
// Save progress
|
|
731
|
+
const progressPath = join(this.investigationsDir, foundInv, 'progress.json');
|
|
732
|
+
await writeFile(progressPath, JSON.stringify(progress, null, 2));
|
|
733
|
+
|
|
734
|
+
const remaining = progress.totalFailed - progress.investigated;
|
|
735
|
+
console.log(`\nā
Marked ${questionId} as investigated`);
|
|
736
|
+
if (category) {
|
|
737
|
+
console.log(` Category: ${category}`);
|
|
738
|
+
}
|
|
739
|
+
console.log(` Progress: ${progress.investigated}/${progress.totalFailed}`);
|
|
740
|
+
console.log(` Remaining: ${remaining}`);
|
|
741
|
+
|
|
742
|
+
if (remaining > 0) {
|
|
743
|
+
console.log(`\n Run: pnpm investigate --next\n`);
|
|
744
|
+
} else {
|
|
745
|
+
console.log(`\nš All questions investigated!`);
|
|
746
|
+
console.log(` Run: pnpm investigate --sync to sync fixes to dataset\n`);
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
private async markFixed(questionId: string): Promise<void> {
|
|
751
|
+
const investigations = await this.listInvestigations();
|
|
752
|
+
|
|
753
|
+
if (investigations.length === 0) {
|
|
754
|
+
console.log('\nš No investigations found.\n');
|
|
755
|
+
return;
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
// Find investigation containing this question
|
|
759
|
+
let foundInv: string | null = null;
|
|
760
|
+
let progress: InvestigationProgress | null = null;
|
|
761
|
+
|
|
762
|
+
for (const inv of investigations) {
|
|
763
|
+
const p = await this.loadProgress(inv);
|
|
764
|
+
if (p && questionId in p.questions) {
|
|
765
|
+
foundInv = inv;
|
|
766
|
+
progress = p;
|
|
767
|
+
break;
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
if (!foundInv || !progress) {
|
|
772
|
+
console.log(`\nā Question ${questionId} not found in any investigation.\n`);
|
|
773
|
+
return;
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
const currentStatus = progress.questions[questionId]?.status;
|
|
777
|
+
if (currentStatus === 'pending') {
|
|
778
|
+
console.log(`\nā ļø Question ${questionId} hasn't been investigated yet.`);
|
|
779
|
+
console.log(` Run: pnpm investigate --done ${questionId} first\n`);
|
|
780
|
+
return;
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
// Update status to fix-implemented
|
|
784
|
+
progress.questions[questionId] = {
|
|
785
|
+
...progress.questions[questionId],
|
|
786
|
+
status: 'fix-implemented',
|
|
787
|
+
};
|
|
788
|
+
|
|
789
|
+
// Save progress
|
|
790
|
+
const progressPath = join(this.investigationsDir, foundInv, 'progress.json');
|
|
791
|
+
await writeFile(progressPath, JSON.stringify(progress, null, 2));
|
|
792
|
+
|
|
793
|
+
const fixedCount = Object.values(progress.questions).filter(
|
|
794
|
+
q => q.status === 'fix-implemented' || q.status === 'synced',
|
|
795
|
+
).length;
|
|
796
|
+
|
|
797
|
+
console.log(`\nā
Marked ${questionId} as fix-implemented`);
|
|
798
|
+
console.log(` Fixed: ${fixedCount}/${progress.totalFailed}`);
|
|
799
|
+
console.log(`\n When ready, run: pnpm investigate --sync\n`);
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
// --------------------------------------------------------------------------
|
|
803
|
+
// Sync to Dataset
|
|
804
|
+
// --------------------------------------------------------------------------
|
|
805
|
+
|
|
806
|
+
private async syncToDataset(): Promise<void> {
|
|
807
|
+
const investigations = await this.listInvestigations();
|
|
808
|
+
|
|
809
|
+
if (investigations.length === 0) {
|
|
810
|
+
console.log('\nš No investigations found.\n');
|
|
811
|
+
return;
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
// Use most recent investigation
|
|
815
|
+
const latestInv = investigations[investigations.length - 1];
|
|
816
|
+
const progress = await this.loadProgress(latestInv);
|
|
817
|
+
|
|
818
|
+
if (!progress) {
|
|
819
|
+
console.log('\nā Could not load progress.\n');
|
|
820
|
+
return;
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
// Load dataset
|
|
824
|
+
const datasetPath = join(this.datasetDir, `${progress.dataset}.json`);
|
|
825
|
+
if (!existsSync(datasetPath)) {
|
|
826
|
+
console.log(`\nā Dataset not found: ${datasetPath}\n`);
|
|
827
|
+
return;
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
const dataset = JSON.parse(await readFile(datasetPath, 'utf-8')) as LongMemEvalQuestion[];
|
|
831
|
+
const datasetMap = new Map<string, LongMemEvalQuestion>();
|
|
832
|
+
for (const q of dataset) {
|
|
833
|
+
datasetMap.set(q.question_id, q);
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
let synced = 0;
|
|
837
|
+
let skipped = 0;
|
|
838
|
+
|
|
839
|
+
console.log(`\nš Syncing fixes to ${progress.dataset}.json...\n`);
|
|
840
|
+
|
|
841
|
+
for (const [questionId, qProgress] of Object.entries(progress.questions)) {
|
|
842
|
+
if (qProgress.status === 'pending') {
|
|
843
|
+
continue;
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
const analysisPath = join(this.investigationsDir, latestInv, questionId, 'analysis.md');
|
|
847
|
+
if (!existsSync(analysisPath)) {
|
|
848
|
+
continue;
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
const content = await readFile(analysisPath, 'utf-8');
|
|
852
|
+
const fixes = this.extractFixes(content);
|
|
853
|
+
|
|
854
|
+
if (!fixes.improved_question && !fixes.improved_answer && !fixes.improvement_note) {
|
|
855
|
+
skipped++;
|
|
856
|
+
continue;
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
const question = datasetMap.get(questionId);
|
|
860
|
+
if (!question) {
|
|
861
|
+
console.warn(` ā ļø Question ${questionId} not found in dataset`);
|
|
862
|
+
continue;
|
|
863
|
+
}
|
|
864
|
+
|
|
865
|
+
// Update question
|
|
866
|
+
let updated = false;
|
|
867
|
+
if (fixes.improved_question && fixes.improved_question !== question.improved_question) {
|
|
868
|
+
question.improved_question = fixes.improved_question;
|
|
869
|
+
updated = true;
|
|
870
|
+
}
|
|
871
|
+
if (fixes.improved_answer && fixes.improved_answer !== question.improved_answer) {
|
|
872
|
+
question.improved_answer = fixes.improved_answer;
|
|
873
|
+
updated = true;
|
|
874
|
+
}
|
|
875
|
+
if (fixes.improvement_note && fixes.improvement_note !== question.improvement_note) {
|
|
876
|
+
question.improvement_note = fixes.improvement_note;
|
|
877
|
+
updated = true;
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
if (updated) {
|
|
881
|
+
console.log(` ā ${questionId}`);
|
|
882
|
+
synced++;
|
|
883
|
+
|
|
884
|
+
// Update progress status
|
|
885
|
+
progress.questions[questionId].status = 'synced';
|
|
886
|
+
} else {
|
|
887
|
+
skipped++;
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
|
|
891
|
+
// Save dataset with 4-space indentation
|
|
892
|
+
await writeFile(datasetPath, JSON.stringify(dataset, null, 4));
|
|
893
|
+
|
|
894
|
+
// Save progress
|
|
895
|
+
const progressPath = join(this.investigationsDir, latestInv, 'progress.json');
|
|
896
|
+
await writeFile(progressPath, JSON.stringify(progress, null, 2));
|
|
897
|
+
|
|
898
|
+
console.log(`\nā
Sync complete!`);
|
|
899
|
+
console.log(` Synced: ${synced}`);
|
|
900
|
+
console.log(` Skipped: ${skipped} (no changes or pending)`);
|
|
901
|
+
console.log(`\nš” Don't forget to run: pnpm run sync-improved-om-qa\n`);
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
// --------------------------------------------------------------------------
|
|
905
|
+
// Helpers
|
|
906
|
+
// --------------------------------------------------------------------------
|
|
907
|
+
|
|
908
|
+
private async findFailures(runIdOrPath: string): Promise<{ failuresPath: string; failures: FailuresFile }> {
|
|
909
|
+
// Check if it's a direct path
|
|
910
|
+
if (existsSync(runIdOrPath)) {
|
|
911
|
+
const content = await readFile(runIdOrPath, 'utf-8');
|
|
912
|
+
return { failuresPath: runIdOrPath, failures: JSON.parse(content) };
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
// Search in results directory
|
|
916
|
+
const configs = await readdir(this.resultsDir).catch(() => []);
|
|
917
|
+
|
|
918
|
+
for (const config of configs) {
|
|
919
|
+
const configDir = join(this.resultsDir, config);
|
|
920
|
+
const runs = await readdir(configDir).catch(() => []);
|
|
921
|
+
|
|
922
|
+
for (const run of runs) {
|
|
923
|
+
if (run === runIdOrPath || run.includes(runIdOrPath)) {
|
|
924
|
+
const failuresPath = join(configDir, run, 'failures.json');
|
|
925
|
+
if (existsSync(failuresPath)) {
|
|
926
|
+
const content = await readFile(failuresPath, 'utf-8');
|
|
927
|
+
return { failuresPath, failures: JSON.parse(content) };
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
throw new Error(`Could not find failures.json for: ${runIdOrPath}`);
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
private async listInvestigations(): Promise<string[]> {
|
|
937
|
+
if (!existsSync(this.investigationsDir)) {
|
|
938
|
+
return [];
|
|
939
|
+
}
|
|
940
|
+
|
|
941
|
+
const entries = await readdir(this.investigationsDir);
|
|
942
|
+
const investigations: string[] = [];
|
|
943
|
+
|
|
944
|
+
for (const entry of entries) {
|
|
945
|
+
const progressPath = join(this.investigationsDir, entry, 'progress.json');
|
|
946
|
+
if (existsSync(progressPath)) {
|
|
947
|
+
investigations.push(entry);
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
return investigations.sort();
|
|
952
|
+
}
|
|
953
|
+
|
|
954
|
+
private async loadProgress(investigation: string): Promise<InvestigationProgress | null> {
|
|
955
|
+
const progressPath = join(this.investigationsDir, investigation, 'progress.json');
|
|
956
|
+
if (!existsSync(progressPath)) {
|
|
957
|
+
return null;
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
return JSON.parse(await readFile(progressPath, 'utf-8'));
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
private extractCategory(content: string): FailureCategory | undefined {
|
|
964
|
+
const categoryMap: Record<string, FailureCategory> = {
|
|
965
|
+
'Observer missed critical information': 'observer-miss',
|
|
966
|
+
'Reflector lost/merged information incorrectly': 'reflector-loss',
|
|
967
|
+
'Agent reasoning error': 'agent-reasoning',
|
|
968
|
+
'Ambiguous/poorly-worded question': 'dataset-error', // Ambiguous questions are dataset errors
|
|
969
|
+
'Dataset inconsistency/error': 'dataset-error',
|
|
970
|
+
'RAG retrieval miss': 'rag-miss',
|
|
971
|
+
};
|
|
972
|
+
|
|
973
|
+
for (const [text, category] of Object.entries(categoryMap)) {
|
|
974
|
+
// Look for checked checkbox
|
|
975
|
+
const pattern = new RegExp(`\\[x\\]\\s*${text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}`, 'i');
|
|
976
|
+
if (pattern.test(content)) {
|
|
977
|
+
return category;
|
|
978
|
+
}
|
|
979
|
+
}
|
|
980
|
+
|
|
981
|
+
// Check for "Other"
|
|
982
|
+
if (/\[x\]\s*Other:/i.test(content)) {
|
|
983
|
+
return 'other';
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
return undefined;
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
private extractFixes(content: string): {
|
|
990
|
+
improved_question?: string;
|
|
991
|
+
improved_answer?: string;
|
|
992
|
+
improvement_note?: string;
|
|
993
|
+
} {
|
|
994
|
+
const fixes: {
|
|
995
|
+
improved_question?: string;
|
|
996
|
+
improved_answer?: string;
|
|
997
|
+
improvement_note?: string;
|
|
998
|
+
} = {};
|
|
999
|
+
|
|
1000
|
+
// Extract improved_question
|
|
1001
|
+
const iqMatch = content.match(/\*\*improved_question\*\*:\s*(.+?)(?:\n|$)/);
|
|
1002
|
+
if (iqMatch && iqMatch[1].trim() && !iqMatch[1].trim().startsWith('<!--')) {
|
|
1003
|
+
fixes.improved_question = iqMatch[1].trim();
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
// Extract improved_answer
|
|
1007
|
+
const iaMatch = content.match(/\*\*improved_answer\*\*:\s*(.+?)(?:\n|$)/);
|
|
1008
|
+
if (iaMatch && iaMatch[1].trim() && !iaMatch[1].trim().startsWith('<!--')) {
|
|
1009
|
+
fixes.improved_answer = iaMatch[1].trim();
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
// Extract improvement_note
|
|
1013
|
+
const inMatch = content.match(/\*\*improvement_note\*\*:\s*(.+?)(?:\n|$)/);
|
|
1014
|
+
if (inMatch && inMatch[1].trim() && !inMatch[1].trim().startsWith('<!--')) {
|
|
1015
|
+
fixes.improvement_note = inMatch[1].trim();
|
|
1016
|
+
}
|
|
1017
|
+
|
|
1018
|
+
return fixes;
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
// --------------------------------------------------------------------------
|
|
1022
|
+
// Investigation Utilities
|
|
1023
|
+
// --------------------------------------------------------------------------
|
|
1024
|
+
|
|
1025
|
+
/**
|
|
1026
|
+
* Inspect a question's data - shows observations, messages, and metadata
|
|
1027
|
+
*/
|
|
1028
|
+
private async inspectQuestion(questionId: string): Promise<void> {
|
|
1029
|
+
console.log(`\nš Inspecting question: ${questionId}\n`);
|
|
1030
|
+
|
|
1031
|
+
// Find the question in prepared data
|
|
1032
|
+
const { omPath, config, dataset } = await this.findPreparedData(questionId);
|
|
1033
|
+
|
|
1034
|
+
if (!omPath) {
|
|
1035
|
+
console.log(`ā No prepared data found for question ${questionId}`);
|
|
1036
|
+
return;
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
console.log(`š Found in: ${dataset}/${config}`);
|
|
1040
|
+
|
|
1041
|
+
// Load using storage adapter
|
|
1042
|
+
const storage = new PersistableInMemoryMemory({ readOnly: true });
|
|
1043
|
+
await storage.hydrate(omPath);
|
|
1044
|
+
const stats = storage.getStats();
|
|
1045
|
+
|
|
1046
|
+
console.log(`\nš Storage Stats:`);
|
|
1047
|
+
console.log(` Threads: ${stats.threads}`);
|
|
1048
|
+
console.log(` Messages: ${stats.messages}`);
|
|
1049
|
+
console.log(` OM Records: ${stats.observationalMemoryRecords}`);
|
|
1050
|
+
|
|
1051
|
+
// Get OM records - use resource scope (null threadId)
|
|
1052
|
+
// Note: resourceId in storage is prefixed with "resource_"
|
|
1053
|
+
const resourceId = `resource_${questionId}`;
|
|
1054
|
+
const omRecords = await storage.getObservationalMemoryHistory(null, resourceId);
|
|
1055
|
+
|
|
1056
|
+
if (omRecords.length === 0) {
|
|
1057
|
+
console.log(`\nā ļø No observational memory records found`);
|
|
1058
|
+
return;
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
const latestRecord = omRecords[0]; // Most recent is first
|
|
1062
|
+
|
|
1063
|
+
console.log(`\nš Latest OM Record:`);
|
|
1064
|
+
console.log(` Created: ${latestRecord.createdAt}`);
|
|
1065
|
+
console.log(` Last Observed: ${latestRecord.lastObservedAt}`);
|
|
1066
|
+
|
|
1067
|
+
// Show observation stats
|
|
1068
|
+
const observations = latestRecord.activeObservations || '';
|
|
1069
|
+
const lines = observations.split('\n').filter((l: string) => l.trim());
|
|
1070
|
+
const dateHeaders = lines.filter((l: string) => /^Date:/i.test(l.trim()));
|
|
1071
|
+
const threadHeaders = lines.filter((l: string) => /<thread\s+id=/i.test(l));
|
|
1072
|
+
|
|
1073
|
+
console.log(`\nš Observations:`);
|
|
1074
|
+
console.log(` Total lines: ${lines.length}`);
|
|
1075
|
+
console.log(` Date groups: ${dateHeaders.length}`);
|
|
1076
|
+
console.log(` Thread sections: ${threadHeaders.length}`);
|
|
1077
|
+
|
|
1078
|
+
// Show first few and last few lines
|
|
1079
|
+
console.log(`\n First 5 lines:`);
|
|
1080
|
+
lines.slice(0, 5).forEach((l: string) => console.log(` ${l.substring(0, 100)}${l.length > 100 ? '...' : ''}`));
|
|
1081
|
+
|
|
1082
|
+
if (lines.length > 10) {
|
|
1083
|
+
console.log(`\n Last 5 lines:`);
|
|
1084
|
+
lines.slice(-5).forEach((l: string) => console.log(` ${l.substring(0, 100)}${l.length > 100 ? '...' : ''}`));
|
|
1085
|
+
}
|
|
1086
|
+
|
|
1087
|
+
console.log(`\nš” Use --search to find specific content:`);
|
|
1088
|
+
console.log(` pnpm investigate --search "sneaker" -q ${questionId}`);
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
/**
|
|
1092
|
+
* Search observations for keywords
|
|
1093
|
+
*/
|
|
1094
|
+
private async searchObservations(questionId: string, keyword: string): Promise<void> {
|
|
1095
|
+
console.log(`\nš Searching for "${keyword}" in question ${questionId}\n`);
|
|
1096
|
+
|
|
1097
|
+
const { omPath } = await this.findPreparedData(questionId);
|
|
1098
|
+
|
|
1099
|
+
if (!omPath) {
|
|
1100
|
+
console.log(`ā No prepared data found for question ${questionId}`);
|
|
1101
|
+
return;
|
|
1102
|
+
}
|
|
1103
|
+
|
|
1104
|
+
// Load using storage adapter
|
|
1105
|
+
const storage = new PersistableInMemoryMemory({ readOnly: true });
|
|
1106
|
+
await storage.hydrate(omPath);
|
|
1107
|
+
|
|
1108
|
+
// Note: resourceId in storage is prefixed with "resource_"
|
|
1109
|
+
const resourceId = `resource_${questionId}`;
|
|
1110
|
+
|
|
1111
|
+
// Search in observations
|
|
1112
|
+
const omRecords = await storage.getObservationalMemoryHistory(null, resourceId);
|
|
1113
|
+
const latestRecord = omRecords[0];
|
|
1114
|
+
const observations = latestRecord?.activeObservations || '';
|
|
1115
|
+
|
|
1116
|
+
const keywordLower = keyword.toLowerCase();
|
|
1117
|
+
const obsLines = observations.split('\n');
|
|
1118
|
+
const obsMatches = obsLines
|
|
1119
|
+
.map((line: string, idx: number) => ({ line, idx }))
|
|
1120
|
+
.filter(({ line }: { line: string }) => line.toLowerCase().includes(keywordLower));
|
|
1121
|
+
|
|
1122
|
+
console.log(`š Observations (${obsMatches.length} matches):`);
|
|
1123
|
+
if (obsMatches.length === 0) {
|
|
1124
|
+
console.log(` No matches found in observations`);
|
|
1125
|
+
} else {
|
|
1126
|
+
for (const { line, idx } of obsMatches) {
|
|
1127
|
+
// Highlight the keyword
|
|
1128
|
+
const highlighted = line.replace(new RegExp(`(${keyword})`, 'gi'), '\x1b[33m$1\x1b[0m');
|
|
1129
|
+
console.log(` [${idx}] ${highlighted}`);
|
|
1130
|
+
}
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
// Search in raw messages
|
|
1134
|
+
const messagesResult = await storage.listMessagesByResourceId({ resourceId, perPage: false });
|
|
1135
|
+
const messages = messagesResult.messages;
|
|
1136
|
+
const msgMatches = messages.filter((m: any) => {
|
|
1137
|
+
const content = typeof m.content === 'string' ? m.content : JSON.stringify(m.content);
|
|
1138
|
+
return content.toLowerCase().includes(keywordLower);
|
|
1139
|
+
});
|
|
1140
|
+
|
|
1141
|
+
console.log(`\nš¬ Raw Messages (${msgMatches.length} matches):`);
|
|
1142
|
+
if (msgMatches.length === 0) {
|
|
1143
|
+
console.log(` No matches found in raw messages`);
|
|
1144
|
+
} else {
|
|
1145
|
+
for (const msg of msgMatches.slice(0, 10)) {
|
|
1146
|
+
const content = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content);
|
|
1147
|
+
const highlighted = content.replace(new RegExp(`(${keyword})`, 'gi'), '\x1b[33m$1\x1b[0m');
|
|
1148
|
+
const date = msg.createdAt ? new Date(msg.createdAt).toISOString().split('T')[0] : 'unknown';
|
|
1149
|
+
console.log(` [${date}] ${msg.role}: ${highlighted.substring(0, 120)}${content.length > 120 ? '...' : ''}`);
|
|
1150
|
+
}
|
|
1151
|
+
if (msgMatches.length > 10) {
|
|
1152
|
+
console.log(` ... and ${msgMatches.length - 10} more`);
|
|
1153
|
+
}
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1156
|
+
// Summary
|
|
1157
|
+
const inObs = obsMatches.length > 0;
|
|
1158
|
+
const inRaw = msgMatches.length > 0;
|
|
1159
|
+
|
|
1160
|
+
console.log(`\nš Summary:`);
|
|
1161
|
+
if (inRaw && !inObs) {
|
|
1162
|
+
console.log(` ā ļø Found in raw messages but NOT in observations`);
|
|
1163
|
+
console.log(` ā Likely an Observer miss`);
|
|
1164
|
+
} else if (inObs && inRaw) {
|
|
1165
|
+
console.log(` ā Found in both raw messages and observations`);
|
|
1166
|
+
} else if (!inRaw && !inObs) {
|
|
1167
|
+
console.log(` ā Not found in raw data or observations`);
|
|
1168
|
+
}
|
|
1169
|
+
}
|
|
1170
|
+
|
|
1171
|
+
/**
|
|
1172
|
+
* Search original dataset for a keyword and show full context
|
|
1173
|
+
*/
|
|
1174
|
+
private async searchOriginalDataset(questionId: string, keyword: string): Promise<void> {
|
|
1175
|
+
console.log(`\nš Searching original dataset for "${keyword}" in question ${questionId}\n`);
|
|
1176
|
+
|
|
1177
|
+
const loader = new DatasetLoader(this.datasetDir);
|
|
1178
|
+
const dataset = await loader.loadDataset('longmemeval_s');
|
|
1179
|
+
const question = dataset.find(q => q.question_id === questionId);
|
|
1180
|
+
|
|
1181
|
+
if (!question?.haystack_sessions || !question?.haystack_dates) {
|
|
1182
|
+
console.log(`ā Question not found or has no sessions`);
|
|
1183
|
+
return;
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
const keywordLower = keyword.toLowerCase();
|
|
1187
|
+
let totalMatches = 0;
|
|
1188
|
+
|
|
1189
|
+
for (let sessionIdx = 0; sessionIdx < question.haystack_sessions.length; sessionIdx++) {
|
|
1190
|
+
const session = question.haystack_sessions[sessionIdx];
|
|
1191
|
+
const date = question.haystack_dates[sessionIdx] || 'unknown';
|
|
1192
|
+
|
|
1193
|
+
for (let turnIdx = 0; turnIdx < session.length; turnIdx++) {
|
|
1194
|
+
const turn = session[turnIdx];
|
|
1195
|
+
const content = turn.content || '';
|
|
1196
|
+
|
|
1197
|
+
if (content.toLowerCase().includes(keywordLower)) {
|
|
1198
|
+
totalMatches++;
|
|
1199
|
+
const roleColor = turn.role === 'user' ? '\x1b[36m' : '\x1b[90m';
|
|
1200
|
+
const reset = '\x1b[0m';
|
|
1201
|
+
const yellow = '\x1b[33m';
|
|
1202
|
+
|
|
1203
|
+
// Highlight the keyword
|
|
1204
|
+
const highlighted = content.replace(new RegExp(`(${keyword})`, 'gi'), `${yellow}$1${reset}`);
|
|
1205
|
+
|
|
1206
|
+
console.log(`${'ā'.repeat(70)}`);
|
|
1207
|
+
console.log(`š
Session ${sessionIdx}, Turn ${turnIdx} | ${date}`);
|
|
1208
|
+
console.log(`${roleColor}[${turn.role.toUpperCase()}]:${reset}`);
|
|
1209
|
+
console.log(highlighted);
|
|
1210
|
+
console.log();
|
|
1211
|
+
}
|
|
1212
|
+
}
|
|
1213
|
+
}
|
|
1214
|
+
|
|
1215
|
+
if (totalMatches === 0) {
|
|
1216
|
+
console.log(`ā No matches found for "${keyword}" in original dataset`);
|
|
1217
|
+
} else {
|
|
1218
|
+
console.log(`${'ā'.repeat(70)}`);
|
|
1219
|
+
console.log(`\nš Total: ${totalMatches} matches across ${question.haystack_sessions.length} sessions`);
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
/**
|
|
1224
|
+
* Trace information flow for a keyword through the pipeline
|
|
1225
|
+
*/
|
|
1226
|
+
private async traceInformation(questionId: string, keyword: string): Promise<void> {
|
|
1227
|
+
console.log(`\nš Tracing "${keyword}" through pipeline for ${questionId}\n`);
|
|
1228
|
+
|
|
1229
|
+
const { omPath, config, dataset, questionDir } = await this.findPreparedData(questionId);
|
|
1230
|
+
|
|
1231
|
+
if (!omPath) {
|
|
1232
|
+
console.log(`ā No prepared data found for question ${questionId}`);
|
|
1233
|
+
return;
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1236
|
+
const keywordLower = keyword.toLowerCase();
|
|
1237
|
+
|
|
1238
|
+
// 1. Check original dataset
|
|
1239
|
+
console.log(`\n1ļøā£ Original Dataset:`);
|
|
1240
|
+
const loader = new DatasetLoader(this.datasetDir);
|
|
1241
|
+
const datasetName = dataset as 'longmemeval_s' | 'longmemeval_m';
|
|
1242
|
+
const questions = await loader.loadDataset(datasetName);
|
|
1243
|
+
const question = questions.find(q => q.question_id === questionId);
|
|
1244
|
+
|
|
1245
|
+
if (question) {
|
|
1246
|
+
// Search in haystack sessions
|
|
1247
|
+
let sessionMatches = 0;
|
|
1248
|
+
const matchingSessions: { idx: number; turnIdx: number; role: string; preview: string }[] = [];
|
|
1249
|
+
|
|
1250
|
+
if (question.haystack_sessions) {
|
|
1251
|
+
for (let sIdx = 0; sIdx < question.haystack_sessions.length; sIdx++) {
|
|
1252
|
+
const session = question.haystack_sessions[sIdx];
|
|
1253
|
+
for (let tIdx = 0; tIdx < session.length; tIdx++) {
|
|
1254
|
+
const turn = session[tIdx];
|
|
1255
|
+
if (turn.content.toLowerCase().includes(keywordLower)) {
|
|
1256
|
+
sessionMatches++;
|
|
1257
|
+
if (matchingSessions.length < 5) {
|
|
1258
|
+
matchingSessions.push({
|
|
1259
|
+
idx: sIdx,
|
|
1260
|
+
turnIdx: tIdx,
|
|
1261
|
+
role: turn.role,
|
|
1262
|
+
preview: turn.content.substring(0, 100),
|
|
1263
|
+
});
|
|
1264
|
+
}
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
}
|
|
1268
|
+
}
|
|
1269
|
+
|
|
1270
|
+
console.log(` Found in ${sessionMatches} session turns`);
|
|
1271
|
+
for (const m of matchingSessions) {
|
|
1272
|
+
const highlighted = m.preview.replace(new RegExp(`(${keyword})`, 'gi'), '\x1b[33m$1\x1b[0m');
|
|
1273
|
+
console.log(` Session ${m.idx}, Turn ${m.turnIdx} (${m.role}): ${highlighted}...`);
|
|
1274
|
+
}
|
|
1275
|
+
if (sessionMatches > 5) {
|
|
1276
|
+
console.log(` ... and ${sessionMatches - 5} more`);
|
|
1277
|
+
}
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1280
|
+
// 2. Check stored messages
|
|
1281
|
+
console.log(`\n2ļøā£ Stored Messages (om.json):`);
|
|
1282
|
+
const storage = new PersistableInMemoryMemory({ readOnly: true });
|
|
1283
|
+
await storage.hydrate(omPath);
|
|
1284
|
+
|
|
1285
|
+
// Note: resourceId in storage is prefixed with "resource_"
|
|
1286
|
+
const resourceId = `resource_${questionId}`;
|
|
1287
|
+
|
|
1288
|
+
const messagesResult = await storage.listMessagesByResourceId({ resourceId, perPage: false });
|
|
1289
|
+
const allMessages = messagesResult.messages;
|
|
1290
|
+
const msgMatches = allMessages.filter((m: any) => {
|
|
1291
|
+
const content = typeof m.content === 'string' ? m.content : JSON.stringify(m.content);
|
|
1292
|
+
return content.toLowerCase().includes(keywordLower);
|
|
1293
|
+
});
|
|
1294
|
+
|
|
1295
|
+
console.log(` Found in ${msgMatches.length} messages`);
|
|
1296
|
+
for (const msg of msgMatches.slice(0, 3)) {
|
|
1297
|
+
const content = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content);
|
|
1298
|
+
const date = msg.createdAt ? new Date(msg.createdAt).toISOString() : 'unknown';
|
|
1299
|
+
console.log(` [${date}] ${msg.role}: ${content.substring(0, 80)}...`);
|
|
1300
|
+
}
|
|
1301
|
+
|
|
1302
|
+
// 3. Check observations
|
|
1303
|
+
console.log(`\n3ļøā£ Observations (activeObservations):`);
|
|
1304
|
+
const omRecords = await storage.getObservationalMemoryHistory(null, resourceId);
|
|
1305
|
+
const latestRecord = omRecords[0]; // Most recent is first
|
|
1306
|
+
const observations = latestRecord?.activeObservations || '';
|
|
1307
|
+
|
|
1308
|
+
const obsLines = observations.split('\n');
|
|
1309
|
+
const obsMatches = obsLines.filter((l: string) => l.toLowerCase().includes(keywordLower));
|
|
1310
|
+
|
|
1311
|
+
console.log(` Found in ${obsMatches.length} observation lines`);
|
|
1312
|
+
for (const line of obsMatches.slice(0, 5)) {
|
|
1313
|
+
const highlighted = line.replace(new RegExp(`(${keyword})`, 'gi'), '\x1b[33m$1\x1b[0m');
|
|
1314
|
+
console.log(` ${highlighted.substring(0, 100)}...`);
|
|
1315
|
+
}
|
|
1316
|
+
|
|
1317
|
+
// 4. Check agent context (om.md)
|
|
1318
|
+
console.log(`\n4ļøā£ Agent Context (om.md):`);
|
|
1319
|
+
const omMdPath = join(questionDir, 'om.md');
|
|
1320
|
+
if (existsSync(omMdPath)) {
|
|
1321
|
+
const omMd = await readFile(omMdPath, 'utf-8');
|
|
1322
|
+
const mdLines = omMd.split('\n');
|
|
1323
|
+
const mdMatches = mdLines.filter(l => l.toLowerCase().includes(keywordLower));
|
|
1324
|
+
|
|
1325
|
+
console.log(` Found in ${mdMatches.length} lines`);
|
|
1326
|
+
for (const line of mdMatches.slice(0, 5)) {
|
|
1327
|
+
const highlighted = line.replace(new RegExp(`(${keyword})`, 'gi'), '\x1b[33m$1\x1b[0m');
|
|
1328
|
+
console.log(` ${highlighted.substring(0, 100)}...`);
|
|
1329
|
+
}
|
|
1330
|
+
} else {
|
|
1331
|
+
console.log(` om.md not found (run benchmark first)`);
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1334
|
+
// 5. Diagnosis
|
|
1335
|
+
console.log(`\nš Diagnosis:`);
|
|
1336
|
+
const inDataset =
|
|
1337
|
+
question && question.haystack_sessions?.some(s => s.some(t => t.content.toLowerCase().includes(keywordLower)));
|
|
1338
|
+
const inMessages = msgMatches.length > 0;
|
|
1339
|
+
const inObservations = obsMatches.length > 0;
|
|
1340
|
+
|
|
1341
|
+
if (inDataset && !inMessages) {
|
|
1342
|
+
console.log(` ā Lost during message storage - check prepare.ts`);
|
|
1343
|
+
} else if (inMessages && !inObservations) {
|
|
1344
|
+
console.log(` ā Observer missed this information`);
|
|
1345
|
+
console.log(` ā Check Observer prompts or message batching`);
|
|
1346
|
+
} else if (inObservations) {
|
|
1347
|
+
console.log(` ā Information preserved through pipeline`);
|
|
1348
|
+
console.log(` ā If agent still failed, it's a reasoning error`);
|
|
1349
|
+
} else if (!inDataset) {
|
|
1350
|
+
console.log(` ā ļø Not found in original dataset`);
|
|
1351
|
+
}
|
|
1352
|
+
}
|
|
1353
|
+
|
|
1354
|
+
// --------------------------------------------------------------------------
|
|
1355
|
+
// Date-Based Observation Viewer
|
|
1356
|
+
// --------------------------------------------------------------------------
|
|
1357
|
+
|
|
1358
|
+
/**
|
|
1359
|
+
* View observations and raw messages around a specific date
|
|
1360
|
+
*/
|
|
1361
|
+
private async viewObservationsAroundDate(
|
|
1362
|
+
questionId: string,
|
|
1363
|
+
dateStr: string,
|
|
1364
|
+
contextDays: number = 1,
|
|
1365
|
+
): Promise<void> {
|
|
1366
|
+
console.log(`\nšļø Viewing data around "${dateStr}" for question ${questionId}\n`);
|
|
1367
|
+
|
|
1368
|
+
// Parse the target date
|
|
1369
|
+
const targetDate = this.parseFlexibleDate(dateStr);
|
|
1370
|
+
if (!targetDate) {
|
|
1371
|
+
console.log(`ā Could not parse date: "${dateStr}"`);
|
|
1372
|
+
console.log(` Try formats like: "2023/05/29", "May 29, 2023", "05-29"`);
|
|
1373
|
+
return;
|
|
1374
|
+
}
|
|
1375
|
+
|
|
1376
|
+
const startDate = new Date(targetDate);
|
|
1377
|
+
startDate.setDate(startDate.getDate() - contextDays);
|
|
1378
|
+
const endDate = new Date(targetDate);
|
|
1379
|
+
endDate.setDate(endDate.getDate() + contextDays);
|
|
1380
|
+
|
|
1381
|
+
console.log(`š
Target: ${targetDate.toDateString()}`);
|
|
1382
|
+
console.log(`š
Range: ${startDate.toDateString()} to ${endDate.toDateString()}\n`);
|
|
1383
|
+
|
|
1384
|
+
// Find prepared data
|
|
1385
|
+
const { omPath, questionDir } = await this.findPreparedData(questionId);
|
|
1386
|
+
if (!omPath) {
|
|
1387
|
+
console.log(`ā No prepared data found for question ${questionId}`);
|
|
1388
|
+
return;
|
|
1389
|
+
}
|
|
1390
|
+
|
|
1391
|
+
// Load storage
|
|
1392
|
+
const storage = new PersistableInMemoryMemory({ readOnly: true });
|
|
1393
|
+
await storage.hydrate(omPath);
|
|
1394
|
+
const resourceId = `resource_${questionId}`;
|
|
1395
|
+
|
|
1396
|
+
// 1. Show raw messages in date range
|
|
1397
|
+
console.log(`1ļøā£ Raw Messages in Range:`);
|
|
1398
|
+
console.log(`${'ā'.repeat(60)}`);
|
|
1399
|
+
|
|
1400
|
+
const messagesResult = await storage.listMessagesByResourceId({ resourceId, perPage: false });
|
|
1401
|
+
const allMessages = messagesResult.messages;
|
|
1402
|
+
|
|
1403
|
+
const messagesInRange = allMessages
|
|
1404
|
+
.filter((m: any) => {
|
|
1405
|
+
if (!m.createdAt) return false;
|
|
1406
|
+
const msgDate = new Date(m.createdAt);
|
|
1407
|
+
return msgDate >= startDate && msgDate <= endDate;
|
|
1408
|
+
})
|
|
1409
|
+
.sort((a: any, b: any) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
1410
|
+
|
|
1411
|
+
if (messagesInRange.length === 0) {
|
|
1412
|
+
console.log(` No messages found in this date range`);
|
|
1413
|
+
} else {
|
|
1414
|
+
console.log(` Found ${messagesInRange.length} messages\n`);
|
|
1415
|
+
for (const msg of messagesInRange) {
|
|
1416
|
+
const content = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content);
|
|
1417
|
+
const date = new Date(msg.createdAt);
|
|
1418
|
+
const dateStr = date.toLocaleDateString('en-US', {
|
|
1419
|
+
month: 'short',
|
|
1420
|
+
day: 'numeric',
|
|
1421
|
+
hour: '2-digit',
|
|
1422
|
+
minute: '2-digit',
|
|
1423
|
+
});
|
|
1424
|
+
const isTargetDay = date.toDateString() === targetDate.toDateString();
|
|
1425
|
+
const prefix = isTargetDay ? 'ā ' : ' ';
|
|
1426
|
+
const highlight = isTargetDay ? '\x1b[33m' : '\x1b[90m';
|
|
1427
|
+
const reset = '\x1b[0m';
|
|
1428
|
+
|
|
1429
|
+
console.log(`${prefix}${highlight}[${dateStr}] ${msg.role}:${reset}`);
|
|
1430
|
+
console.log(`${prefix}${highlight}${content.substring(0, 200)}${content.length > 200 ? '...' : ''}${reset}\n`);
|
|
1431
|
+
}
|
|
1432
|
+
}
|
|
1433
|
+
|
|
1434
|
+
// 2. Show observations for dates in range
|
|
1435
|
+
console.log(`\n2ļøā£ Observations in Range:`);
|
|
1436
|
+
console.log(`${'ā'.repeat(60)}`);
|
|
1437
|
+
|
|
1438
|
+
const omRecords = await storage.getObservationalMemoryHistory(null, resourceId);
|
|
1439
|
+
const latestRecord = omRecords[0];
|
|
1440
|
+
const observations = latestRecord?.activeObservations || '';
|
|
1441
|
+
|
|
1442
|
+
if (!observations) {
|
|
1443
|
+
console.log(` No observations found`);
|
|
1444
|
+
} else {
|
|
1445
|
+
// Parse observations by date
|
|
1446
|
+
const lines = observations.split('\n');
|
|
1447
|
+
let currentDateGroup = '';
|
|
1448
|
+
let inRange = false;
|
|
1449
|
+
let observationsInRange: { date: string; lines: string[] }[] = [];
|
|
1450
|
+
let currentGroup: { date: string; lines: string[] } | null = null;
|
|
1451
|
+
|
|
1452
|
+
for (const line of lines) {
|
|
1453
|
+
// Check for date headers like "Date: May 29, 2023" or "Date: Monday, May 29, 2023"
|
|
1454
|
+
const dateMatch = line.match(/^Date:\s*(.+)$/i);
|
|
1455
|
+
if (dateMatch) {
|
|
1456
|
+
// Save previous group if in range
|
|
1457
|
+
if (currentGroup && currentGroup.lines.length > 0) {
|
|
1458
|
+
observationsInRange.push(currentGroup);
|
|
1459
|
+
}
|
|
1460
|
+
|
|
1461
|
+
currentDateGroup = dateMatch[1].trim();
|
|
1462
|
+
const parsedDate = this.parseFlexibleDate(currentDateGroup);
|
|
1463
|
+
inRange = parsedDate ? parsedDate >= startDate && parsedDate <= endDate : false;
|
|
1464
|
+
|
|
1465
|
+
if (inRange) {
|
|
1466
|
+
currentGroup = { date: currentDateGroup, lines: [] };
|
|
1467
|
+
} else {
|
|
1468
|
+
currentGroup = null;
|
|
1469
|
+
}
|
|
1470
|
+
} else if (inRange && currentGroup && line.trim()) {
|
|
1471
|
+
// Skip thread tags but include observation content
|
|
1472
|
+
if (!line.match(/^<\/?thread/)) {
|
|
1473
|
+
currentGroup.lines.push(line);
|
|
1474
|
+
}
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
|
|
1478
|
+
// Don't forget the last group
|
|
1479
|
+
if (currentGroup && currentGroup.lines.length > 0) {
|
|
1480
|
+
observationsInRange.push(currentGroup);
|
|
1481
|
+
}
|
|
1482
|
+
|
|
1483
|
+
if (observationsInRange.length === 0) {
|
|
1484
|
+
console.log(` No observations found for dates in range`);
|
|
1485
|
+
} else {
|
|
1486
|
+
for (const group of observationsInRange) {
|
|
1487
|
+
const parsedDate = this.parseFlexibleDate(group.date);
|
|
1488
|
+
const isTargetDay = parsedDate && parsedDate.toDateString() === targetDate.toDateString();
|
|
1489
|
+
const highlight = isTargetDay ? '\x1b[33m' : '\x1b[0m';
|
|
1490
|
+
const reset = '\x1b[0m';
|
|
1491
|
+
|
|
1492
|
+
console.log(`\n${highlight}š
${group.date}${reset}`);
|
|
1493
|
+
for (const line of group.lines) {
|
|
1494
|
+
console.log(` ${highlight}${line}${reset}`);
|
|
1495
|
+
}
|
|
1496
|
+
}
|
|
1497
|
+
}
|
|
1498
|
+
}
|
|
1499
|
+
|
|
1500
|
+
// 3. Show original dataset sessions for this date
|
|
1501
|
+
console.log(`\n\n3ļøā£ Original Dataset Sessions:`);
|
|
1502
|
+
console.log(`${'ā'.repeat(60)}`);
|
|
1503
|
+
|
|
1504
|
+
const loader = new DatasetLoader(this.datasetDir);
|
|
1505
|
+
const dataset = await loader.loadDataset('longmemeval_s');
|
|
1506
|
+
const question = dataset.find(q => q.question_id === questionId);
|
|
1507
|
+
|
|
1508
|
+
if (question?.haystack_sessions && question?.haystack_dates) {
|
|
1509
|
+
const sessionsInRange: { idx: number; date: string; turns: any[] }[] = [];
|
|
1510
|
+
|
|
1511
|
+
for (let i = 0; i < question.haystack_dates.length; i++) {
|
|
1512
|
+
const sessionDateStr = question.haystack_dates[i];
|
|
1513
|
+
const sessionDate = this.parseFlexibleDate(sessionDateStr);
|
|
1514
|
+
|
|
1515
|
+
if (sessionDate && sessionDate >= startDate && sessionDate <= endDate) {
|
|
1516
|
+
sessionsInRange.push({
|
|
1517
|
+
idx: i,
|
|
1518
|
+
date: sessionDateStr,
|
|
1519
|
+
turns: question.haystack_sessions[i] || [],
|
|
1520
|
+
});
|
|
1521
|
+
}
|
|
1522
|
+
}
|
|
1523
|
+
|
|
1524
|
+
if (sessionsInRange.length === 0) {
|
|
1525
|
+
console.log(` No sessions found in date range`);
|
|
1526
|
+
} else {
|
|
1527
|
+
console.log(` Found ${sessionsInRange.length} sessions\n`);
|
|
1528
|
+
for (const session of sessionsInRange) {
|
|
1529
|
+
const sessionDate = this.parseFlexibleDate(session.date);
|
|
1530
|
+
const isTargetDay = sessionDate && sessionDate.toDateString() === targetDate.toDateString();
|
|
1531
|
+
const highlight = isTargetDay ? '\x1b[33m' : '\x1b[90m';
|
|
1532
|
+
const reset = '\x1b[0m';
|
|
1533
|
+
|
|
1534
|
+
console.log(`${highlight}āāā Session ${session.idx}: ${session.date} āāā${reset}`);
|
|
1535
|
+
for (let t = 0; t < session.turns.length; t++) {
|
|
1536
|
+
const turn = session.turns[t];
|
|
1537
|
+
console.log(`${highlight}[Turn ${t}] ${turn.role}:${reset}`);
|
|
1538
|
+
console.log(
|
|
1539
|
+
`${highlight}${turn.content.substring(0, 300)}${turn.content.length > 300 ? '...' : ''}${reset}\n`,
|
|
1540
|
+
);
|
|
1541
|
+
}
|
|
1542
|
+
}
|
|
1543
|
+
}
|
|
1544
|
+
} else {
|
|
1545
|
+
console.log(` Question not found in dataset`);
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1548
|
+
|
|
1549
|
+
// --------------------------------------------------------------------------
|
|
1550
|
+
// Session Viewer
|
|
1551
|
+
// --------------------------------------------------------------------------
|
|
1552
|
+
|
|
1553
|
+
/**
|
|
1554
|
+
* List all sessions with their dates for a question
|
|
1555
|
+
*/
|
|
1556
|
+
private async listSessions(questionId: string): Promise<void> {
|
|
1557
|
+
console.log(`\nš Sessions for question ${questionId}\n`);
|
|
1558
|
+
|
|
1559
|
+
const loader = new DatasetLoader(this.datasetDir);
|
|
1560
|
+
const dataset = await loader.loadDataset('longmemeval_s');
|
|
1561
|
+
const question = dataset.find(q => q.question_id === questionId);
|
|
1562
|
+
|
|
1563
|
+
if (!question?.haystack_sessions || !question?.haystack_dates) {
|
|
1564
|
+
console.log(`ā Question not found or has no sessions`);
|
|
1565
|
+
return;
|
|
1566
|
+
}
|
|
1567
|
+
|
|
1568
|
+
console.log(`Total sessions: ${question.haystack_sessions.length}\n`);
|
|
1569
|
+
console.log(`${'Idx'.padStart(4)} | ${'Date'.padEnd(25)} | Turns | Preview`);
|
|
1570
|
+
console.log(`${'ā'.repeat(80)}`);
|
|
1571
|
+
|
|
1572
|
+
for (let i = 0; i < question.haystack_sessions.length; i++) {
|
|
1573
|
+
const session = question.haystack_sessions[i];
|
|
1574
|
+
const date = question.haystack_dates[i] || 'unknown';
|
|
1575
|
+
const turns = session.length;
|
|
1576
|
+
const firstUserTurn = session.find(t => t.role === 'user');
|
|
1577
|
+
const preview = firstUserTurn?.content.substring(0, 40) || '';
|
|
1578
|
+
|
|
1579
|
+
console.log(`${String(i).padStart(4)} | ${date.padEnd(25)} | ${String(turns).padStart(5)} | ${preview}...`);
|
|
1580
|
+
}
|
|
1581
|
+
|
|
1582
|
+
console.log(`\nUse --session <idx> -q ${questionId} to view a specific session`);
|
|
1583
|
+
}
|
|
1584
|
+
|
|
1585
|
+
/**
|
|
1586
|
+
* View a specific session from the original dataset
|
|
1587
|
+
*/
|
|
1588
|
+
private async viewSession(questionId: string, sessionIdx: number): Promise<void> {
|
|
1589
|
+
console.log(`\nš Session ${sessionIdx} for question ${questionId}\n`);
|
|
1590
|
+
|
|
1591
|
+
const loader = new DatasetLoader(this.datasetDir);
|
|
1592
|
+
const dataset = await loader.loadDataset('longmemeval_s');
|
|
1593
|
+
const question = dataset.find(q => q.question_id === questionId);
|
|
1594
|
+
|
|
1595
|
+
if (!question?.haystack_sessions || !question?.haystack_dates) {
|
|
1596
|
+
console.log(`ā Question not found or has no sessions`);
|
|
1597
|
+
return;
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
if (sessionIdx < 0 || sessionIdx >= question.haystack_sessions.length) {
|
|
1601
|
+
console.log(`ā Invalid session index. Valid range: 0-${question.haystack_sessions.length - 1}`);
|
|
1602
|
+
return;
|
|
1603
|
+
}
|
|
1604
|
+
|
|
1605
|
+
const session = question.haystack_sessions[sessionIdx];
|
|
1606
|
+
const date = question.haystack_dates[sessionIdx] || 'unknown';
|
|
1607
|
+
|
|
1608
|
+
console.log(`š
Date: ${date}`);
|
|
1609
|
+
console.log(`š¬ Turns: ${session.length}`);
|
|
1610
|
+
console.log(`${'ā'.repeat(60)}\n`);
|
|
1611
|
+
|
|
1612
|
+
for (let t = 0; t < session.length; t++) {
|
|
1613
|
+
const turn = session[t];
|
|
1614
|
+
const roleColor = turn.role === 'user' ? '\x1b[36m' : '\x1b[90m';
|
|
1615
|
+
const reset = '\x1b[0m';
|
|
1616
|
+
|
|
1617
|
+
console.log(`${roleColor}[Turn ${t}] ${turn.role.toUpperCase()}:${reset}`);
|
|
1618
|
+
console.log(`${turn.content}\n`);
|
|
1619
|
+
}
|
|
1620
|
+
|
|
1621
|
+
// Also show what was observed for this date
|
|
1622
|
+
const { omPath } = await this.findPreparedData(questionId);
|
|
1623
|
+
if (omPath) {
|
|
1624
|
+
const storage = new PersistableInMemoryMemory({ readOnly: true });
|
|
1625
|
+
await storage.hydrate(omPath);
|
|
1626
|
+
const resourceId = `resource_${questionId}`;
|
|
1627
|
+
|
|
1628
|
+
const omRecords = await storage.getObservationalMemoryHistory(null, resourceId);
|
|
1629
|
+
const latestRecord = omRecords[0];
|
|
1630
|
+
const observations = latestRecord?.activeObservations || '';
|
|
1631
|
+
|
|
1632
|
+
if (observations) {
|
|
1633
|
+
// Find observations for this date
|
|
1634
|
+
const sessionDate = this.parseFlexibleDate(date);
|
|
1635
|
+
if (sessionDate) {
|
|
1636
|
+
const dateStr = sessionDate.toLocaleDateString('en-US', {
|
|
1637
|
+
weekday: 'long',
|
|
1638
|
+
month: 'long',
|
|
1639
|
+
day: 'numeric',
|
|
1640
|
+
year: 'numeric',
|
|
1641
|
+
});
|
|
1642
|
+
|
|
1643
|
+
console.log(`\n${'ā'.repeat(60)}`);
|
|
1644
|
+
console.log(`\nš Observations extracted for ${dateStr}:\n`);
|
|
1645
|
+
|
|
1646
|
+
const lines = observations.split('\n');
|
|
1647
|
+
let inTargetDate = false;
|
|
1648
|
+
let foundAny = false;
|
|
1649
|
+
|
|
1650
|
+
for (const line of lines) {
|
|
1651
|
+
const dateMatch = line.match(/^Date:\s*(.+)$/i);
|
|
1652
|
+
if (dateMatch) {
|
|
1653
|
+
const lineDate = this.parseFlexibleDate(dateMatch[1].trim());
|
|
1654
|
+
inTargetDate = lineDate?.toDateString() === sessionDate.toDateString();
|
|
1655
|
+
} else if (inTargetDate && line.trim() && !line.match(/^<\/?thread/)) {
|
|
1656
|
+
console.log(` ${line}`);
|
|
1657
|
+
foundAny = true;
|
|
1658
|
+
}
|
|
1659
|
+
}
|
|
1660
|
+
|
|
1661
|
+
if (!foundAny) {
|
|
1662
|
+
console.log(` ā ļø No observations found for this date`);
|
|
1663
|
+
}
|
|
1664
|
+
}
|
|
1665
|
+
}
|
|
1666
|
+
}
|
|
1667
|
+
}
|
|
1668
|
+
|
|
1669
|
+
/**
|
|
1670
|
+
* Parse various date formats flexibly
|
|
1671
|
+
*/
|
|
1672
|
+
private parseFlexibleDate(dateStr: string): Date | null {
|
|
1673
|
+
if (!dateStr) return null;
|
|
1674
|
+
|
|
1675
|
+
// Try various formats
|
|
1676
|
+
const formats = [
|
|
1677
|
+
// "2023/05/29 (Mon) 15:01" - LongMemEval format
|
|
1678
|
+
/^(\d{4})\/(\d{2})\/(\d{2})/,
|
|
1679
|
+
// "May 29, 2023" or "Monday, May 29, 2023"
|
|
1680
|
+
/(?:(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday),?\s*)?(\w+)\s+(\d{1,2}),?\s*(\d{4})/i,
|
|
1681
|
+
// "05-29-2023" or "05/29/2023"
|
|
1682
|
+
/^(\d{2})[-\/](\d{2})[-\/](\d{4})$/,
|
|
1683
|
+
// "2023-05-29"
|
|
1684
|
+
/^(\d{4})-(\d{2})-(\d{2})/,
|
|
1685
|
+
];
|
|
1686
|
+
|
|
1687
|
+
// Try LongMemEval format first: "2023/05/29 (Mon) 15:01"
|
|
1688
|
+
const longMemMatch = dateStr.match(/^(\d{4})\/(\d{2})\/(\d{2})/);
|
|
1689
|
+
if (longMemMatch) {
|
|
1690
|
+
return new Date(parseInt(longMemMatch[1]), parseInt(longMemMatch[2]) - 1, parseInt(longMemMatch[3]));
|
|
1691
|
+
}
|
|
1692
|
+
|
|
1693
|
+
// Try "May 29, 2023" format
|
|
1694
|
+
const monthNameMatch = dateStr.match(
|
|
1695
|
+
/(?:(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday),?\s*)?(\w+)\s+(\d{1,2}),?\s*(\d{4})/i,
|
|
1696
|
+
);
|
|
1697
|
+
if (monthNameMatch) {
|
|
1698
|
+
const months: Record<string, number> = {
|
|
1699
|
+
january: 0,
|
|
1700
|
+
february: 1,
|
|
1701
|
+
march: 2,
|
|
1702
|
+
april: 3,
|
|
1703
|
+
may: 4,
|
|
1704
|
+
june: 5,
|
|
1705
|
+
july: 6,
|
|
1706
|
+
august: 7,
|
|
1707
|
+
september: 8,
|
|
1708
|
+
october: 9,
|
|
1709
|
+
november: 10,
|
|
1710
|
+
december: 11,
|
|
1711
|
+
};
|
|
1712
|
+
const monthNum = months[monthNameMatch[1].toLowerCase()];
|
|
1713
|
+
if (monthNum !== undefined) {
|
|
1714
|
+
return new Date(parseInt(monthNameMatch[3]), monthNum, parseInt(monthNameMatch[2]));
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
|
|
1718
|
+
// Try ISO format: "2023-05-29"
|
|
1719
|
+
const isoMatch = dateStr.match(/^(\d{4})-(\d{2})-(\d{2})/);
|
|
1720
|
+
if (isoMatch) {
|
|
1721
|
+
return new Date(parseInt(isoMatch[1]), parseInt(isoMatch[2]) - 1, parseInt(isoMatch[3]));
|
|
1722
|
+
}
|
|
1723
|
+
|
|
1724
|
+
// Fallback to native Date parsing
|
|
1725
|
+
const parsed = new Date(dateStr);
|
|
1726
|
+
return isNaN(parsed.getTime()) ? null : parsed;
|
|
1727
|
+
}
|
|
1728
|
+
|
|
1729
|
+
// --------------------------------------------------------------------------
|
|
1730
|
+
// Stale Data Detection
|
|
1731
|
+
// --------------------------------------------------------------------------
|
|
1732
|
+
|
|
1733
|
+
/**
|
|
1734
|
+
* Check if prepared data is stale (missing per-thread cursors).
|
|
1735
|
+
* Data prepared before the per-thread cursor fix will be missing
|
|
1736
|
+
* lastObservedAt on thread metadata, which can cause observation gaps.
|
|
1737
|
+
*/
|
|
1738
|
+
private async checkStaleData(questionId?: string, staleOnly?: boolean): Promise<void> {
|
|
1739
|
+
console.log('\nš Checking for stale prepared data (pre-cursor-fix)...\n');
|
|
1740
|
+
|
|
1741
|
+
interface StaleCheckResult {
|
|
1742
|
+
questionId: string;
|
|
1743
|
+
config: string;
|
|
1744
|
+
dataset: string;
|
|
1745
|
+
isStale: boolean;
|
|
1746
|
+
threadCount: number;
|
|
1747
|
+
threadsWithCursor: number;
|
|
1748
|
+
reason?: string;
|
|
1749
|
+
}
|
|
1750
|
+
|
|
1751
|
+
const results: StaleCheckResult[] = [];
|
|
1752
|
+
|
|
1753
|
+
// If specific question, just check that one
|
|
1754
|
+
if (questionId) {
|
|
1755
|
+
const { omPath, config, dataset } = await this.findPreparedData(questionId);
|
|
1756
|
+
if (!omPath) {
|
|
1757
|
+
console.log(`ā No prepared data found for question ${questionId}`);
|
|
1758
|
+
return;
|
|
1759
|
+
}
|
|
1760
|
+
|
|
1761
|
+
const result = await this.checkSingleQuestion(questionId, omPath, config, dataset);
|
|
1762
|
+
results.push(result);
|
|
1763
|
+
} else {
|
|
1764
|
+
// Check all questions in all configs
|
|
1765
|
+
const datasets = ['longmemeval_s', 'longmemeval_m'];
|
|
1766
|
+
|
|
1767
|
+
for (const dataset of datasets) {
|
|
1768
|
+
const datasetDir = join(this.preparedDataDir, dataset);
|
|
1769
|
+
if (!existsSync(datasetDir)) continue;
|
|
1770
|
+
|
|
1771
|
+
const configs = await readdir(datasetDir).catch(() => []);
|
|
1772
|
+
|
|
1773
|
+
for (const config of configs) {
|
|
1774
|
+
// Only check OM configs
|
|
1775
|
+
if (!config.includes('observational-memory') && !config.startsWith('om-')) continue;
|
|
1776
|
+
|
|
1777
|
+
const configDir = join(datasetDir, config);
|
|
1778
|
+
const questions = await readdir(configDir).catch(() => []);
|
|
1779
|
+
|
|
1780
|
+
for (const qId of questions) {
|
|
1781
|
+
const omPath = join(configDir, qId, 'om.json');
|
|
1782
|
+
if (!existsSync(omPath)) continue;
|
|
1783
|
+
|
|
1784
|
+
const result = await this.checkSingleQuestion(qId, omPath, config, dataset);
|
|
1785
|
+
results.push(result);
|
|
1786
|
+
}
|
|
1787
|
+
}
|
|
1788
|
+
}
|
|
1789
|
+
}
|
|
1790
|
+
|
|
1791
|
+
// Display results
|
|
1792
|
+
const staleResults = results.filter(r => r.isStale);
|
|
1793
|
+
const freshResults = results.filter(r => !r.isStale);
|
|
1794
|
+
|
|
1795
|
+
if (staleOnly) {
|
|
1796
|
+
// Only show stale questions
|
|
1797
|
+
if (staleResults.length === 0) {
|
|
1798
|
+
console.log('ā
No stale data found!');
|
|
1799
|
+
return;
|
|
1800
|
+
}
|
|
1801
|
+
|
|
1802
|
+
console.log(`Found ${staleResults.length} stale question(s):\n`);
|
|
1803
|
+
|
|
1804
|
+
// Group by config
|
|
1805
|
+
const byConfig: Record<string, StaleCheckResult[]> = {};
|
|
1806
|
+
for (const r of staleResults) {
|
|
1807
|
+
const key = `${r.dataset}/${r.config}`;
|
|
1808
|
+
if (!byConfig[key]) byConfig[key] = [];
|
|
1809
|
+
byConfig[key].push(r);
|
|
1810
|
+
}
|
|
1811
|
+
|
|
1812
|
+
for (const [configKey, questions] of Object.entries(byConfig)) {
|
|
1813
|
+
console.log(`š ${configKey} (${questions.length} stale):`);
|
|
1814
|
+
for (const q of questions.slice(0, 20)) {
|
|
1815
|
+
console.log(` ${q.questionId} - ${q.reason}`);
|
|
1816
|
+
}
|
|
1817
|
+
if (questions.length > 20) {
|
|
1818
|
+
console.log(` ... and ${questions.length - 20} more`);
|
|
1819
|
+
}
|
|
1820
|
+
console.log();
|
|
1821
|
+
}
|
|
1822
|
+
|
|
1823
|
+
// Output question IDs for re-preparation
|
|
1824
|
+
console.log('\nš Stale question IDs (for re-preparation):');
|
|
1825
|
+
console.log(staleResults.map(r => r.questionId).join('\n'));
|
|
1826
|
+
} else {
|
|
1827
|
+
// Show summary
|
|
1828
|
+
console.log(`š Summary:`);
|
|
1829
|
+
console.log(` Total checked: ${results.length}`);
|
|
1830
|
+
console.log(` ā
Fresh (has per-thread cursors): ${freshResults.length}`);
|
|
1831
|
+
console.log(` ā ļø Stale (missing cursors): ${staleResults.length}`);
|
|
1832
|
+
|
|
1833
|
+
if (staleResults.length > 0) {
|
|
1834
|
+
console.log(`\nā ļø Stale questions may have observation gaps!`);
|
|
1835
|
+
console.log(` Run with --stale-only to list them for re-preparation.`);
|
|
1836
|
+
}
|
|
1837
|
+
}
|
|
1838
|
+
}
|
|
1839
|
+
|
|
1840
|
+
private async checkSingleQuestion(
|
|
1841
|
+
questionId: string,
|
|
1842
|
+
omPath: string,
|
|
1843
|
+
config: string,
|
|
1844
|
+
dataset: string,
|
|
1845
|
+
): Promise<{
|
|
1846
|
+
questionId: string;
|
|
1847
|
+
config: string;
|
|
1848
|
+
dataset: string;
|
|
1849
|
+
isStale: boolean;
|
|
1850
|
+
threadCount: number;
|
|
1851
|
+
threadsWithCursor: number;
|
|
1852
|
+
reason?: string;
|
|
1853
|
+
}> {
|
|
1854
|
+
try {
|
|
1855
|
+
const data = JSON.parse(await readFile(omPath, 'utf-8'));
|
|
1856
|
+
const threads = data.threads || [];
|
|
1857
|
+
|
|
1858
|
+
let threadCount = 0;
|
|
1859
|
+
let threadsWithCursor = 0;
|
|
1860
|
+
|
|
1861
|
+
for (const threadPair of threads) {
|
|
1862
|
+
// Structure is [threadId, threadObject]
|
|
1863
|
+
if (!Array.isArray(threadPair) || threadPair.length < 2) continue;
|
|
1864
|
+
|
|
1865
|
+
const threadObj = threadPair[1];
|
|
1866
|
+
if (typeof threadObj !== 'object') continue;
|
|
1867
|
+
|
|
1868
|
+
threadCount++;
|
|
1869
|
+
const metadata = threadObj.metadata || {};
|
|
1870
|
+
|
|
1871
|
+
// Check both old location (metadata.lastObservedAt) and new location (metadata.mastra.om.lastObservedAt)
|
|
1872
|
+
const hasOldCursor = !!metadata.lastObservedAt;
|
|
1873
|
+
const hasNewCursor = !!metadata?.mastra?.om?.lastObservedAt;
|
|
1874
|
+
|
|
1875
|
+
if (hasOldCursor || hasNewCursor) {
|
|
1876
|
+
threadsWithCursor++;
|
|
1877
|
+
}
|
|
1878
|
+
}
|
|
1879
|
+
|
|
1880
|
+
const isStale = threadCount > 0 && threadsWithCursor === 0;
|
|
1881
|
+
const reason = isStale
|
|
1882
|
+
? `0/${threadCount} threads have lastObservedAt`
|
|
1883
|
+
: threadsWithCursor < threadCount
|
|
1884
|
+
? `${threadsWithCursor}/${threadCount} threads have lastObservedAt`
|
|
1885
|
+
: undefined;
|
|
1886
|
+
|
|
1887
|
+
return {
|
|
1888
|
+
questionId,
|
|
1889
|
+
config,
|
|
1890
|
+
dataset,
|
|
1891
|
+
isStale,
|
|
1892
|
+
threadCount,
|
|
1893
|
+
threadsWithCursor,
|
|
1894
|
+
reason,
|
|
1895
|
+
};
|
|
1896
|
+
} catch (error) {
|
|
1897
|
+
return {
|
|
1898
|
+
questionId,
|
|
1899
|
+
config,
|
|
1900
|
+
dataset,
|
|
1901
|
+
isStale: true,
|
|
1902
|
+
threadCount: 0,
|
|
1903
|
+
threadsWithCursor: 0,
|
|
1904
|
+
reason: `Error reading om.json: ${error}`,
|
|
1905
|
+
};
|
|
1906
|
+
}
|
|
1907
|
+
}
|
|
1908
|
+
|
|
1909
|
+
/**
|
|
1910
|
+
* Find prepared data for a question across all configs
|
|
1911
|
+
*/
|
|
1912
|
+
private async findPreparedData(questionId: string): Promise<{
|
|
1913
|
+
omPath: string | null;
|
|
1914
|
+
config: string;
|
|
1915
|
+
dataset: string;
|
|
1916
|
+
questionDir: string;
|
|
1917
|
+
}> {
|
|
1918
|
+
const datasets = ['longmemeval_s', 'longmemeval_m'];
|
|
1919
|
+
|
|
1920
|
+
for (const dataset of datasets) {
|
|
1921
|
+
const datasetDir = join(this.preparedDataDir, dataset);
|
|
1922
|
+
if (!existsSync(datasetDir)) continue;
|
|
1923
|
+
|
|
1924
|
+
const configs = await readdir(datasetDir).catch(() => []);
|
|
1925
|
+
|
|
1926
|
+
for (const config of configs) {
|
|
1927
|
+
const questionDir = join(datasetDir, config, questionId);
|
|
1928
|
+
const omPath = join(questionDir, 'om.json');
|
|
1929
|
+
|
|
1930
|
+
if (existsSync(omPath)) {
|
|
1931
|
+
return { omPath, config, dataset, questionDir };
|
|
1932
|
+
}
|
|
1933
|
+
}
|
|
1934
|
+
}
|
|
1935
|
+
|
|
1936
|
+
return { omPath: null, config: '', dataset: '', questionDir: '' };
|
|
1937
|
+
}
|
|
1938
|
+
|
|
1939
|
+
// --------------------------------------------------------------------------
|
|
1940
|
+
// Add Improvement to Dataset
|
|
1941
|
+
// --------------------------------------------------------------------------
|
|
1942
|
+
|
|
1943
|
+
private async addImprovement(
|
|
1944
|
+
questionId: string,
|
|
1945
|
+
improvedQuestion?: string,
|
|
1946
|
+
improvedAnswer?: string,
|
|
1947
|
+
improvementNote?: string,
|
|
1948
|
+
category?: FailureCategory,
|
|
1949
|
+
clearImproved?: string | boolean,
|
|
1950
|
+
): Promise<void> {
|
|
1951
|
+
if (!clearImproved && !improvedQuestion && !improvedAnswer && !improvementNote && !category) {
|
|
1952
|
+
console.log(
|
|
1953
|
+
'ā At least one of --improve-question, --improve-answer, --improve-note, --category, or --clear-improved is required',
|
|
1954
|
+
);
|
|
1955
|
+
return;
|
|
1956
|
+
}
|
|
1957
|
+
|
|
1958
|
+
// Parse clearImproved into specific fields to clear
|
|
1959
|
+
const clearFields = new Set<string>();
|
|
1960
|
+
if (clearImproved === true || clearImproved === 'all') {
|
|
1961
|
+
clearFields.add('question');
|
|
1962
|
+
clearFields.add('answer');
|
|
1963
|
+
clearFields.add('note');
|
|
1964
|
+
clearFields.add('category');
|
|
1965
|
+
} else if (typeof clearImproved === 'string') {
|
|
1966
|
+
// Support comma-separated list: "note,category"
|
|
1967
|
+
for (const field of clearImproved.split(',')) {
|
|
1968
|
+
const trimmed = field.trim().toLowerCase();
|
|
1969
|
+
if (['question', 'answer', 'note', 'category'].includes(trimmed)) {
|
|
1970
|
+
clearFields.add(trimmed);
|
|
1971
|
+
} else {
|
|
1972
|
+
console.log(`ā Unknown field to clear: ${trimmed}. Valid fields: question, answer, note, category, all`);
|
|
1973
|
+
return;
|
|
1974
|
+
}
|
|
1975
|
+
}
|
|
1976
|
+
}
|
|
1977
|
+
|
|
1978
|
+
const datasetPath = join(this.datasetDir, 'longmemeval_s.json');
|
|
1979
|
+
|
|
1980
|
+
if (!existsSync(datasetPath)) {
|
|
1981
|
+
console.log(`ā Dataset not found: ${datasetPath}`);
|
|
1982
|
+
return;
|
|
1983
|
+
}
|
|
1984
|
+
|
|
1985
|
+
// Load dataset
|
|
1986
|
+
const datasetContent = await readFile(datasetPath, 'utf-8');
|
|
1987
|
+
const dataset: LongMemEvalQuestion[] = JSON.parse(datasetContent);
|
|
1988
|
+
|
|
1989
|
+
// Find the question
|
|
1990
|
+
const questionIndex = dataset.findIndex(q => q.question_id === questionId);
|
|
1991
|
+
if (questionIndex === -1) {
|
|
1992
|
+
console.log(`ā Question not found: ${questionId}`);
|
|
1993
|
+
return;
|
|
1994
|
+
}
|
|
1995
|
+
|
|
1996
|
+
const question = dataset[questionIndex];
|
|
1997
|
+
|
|
1998
|
+
// Show current state
|
|
1999
|
+
console.log(`\nš Updating question: ${questionId}`);
|
|
2000
|
+
console.log(` Type: ${question.question_type}`);
|
|
2001
|
+
console.log(` Question: ${question.question.substring(0, 80)}...`);
|
|
2002
|
+
console.log(` Expected: ${String(question.answer).substring(0, 80)}`);
|
|
2003
|
+
|
|
2004
|
+
// Show what's being updated
|
|
2005
|
+
console.log('\nš Changes:');
|
|
2006
|
+
|
|
2007
|
+
// Handle clearing specific fields
|
|
2008
|
+
if (clearFields.has('question') && question.improved_question) {
|
|
2009
|
+
console.log(` improved_question: ${question.improved_question} ā (cleared)`);
|
|
2010
|
+
delete question.improved_question;
|
|
2011
|
+
}
|
|
2012
|
+
if (clearFields.has('answer') && question.improved_answer) {
|
|
2013
|
+
console.log(` improved_answer: ${question.improved_answer.substring(0, 50)}... ā (cleared)`);
|
|
2014
|
+
delete question.improved_answer;
|
|
2015
|
+
}
|
|
2016
|
+
if (clearFields.has('note') && question.improvement_note) {
|
|
2017
|
+
console.log(` improvement_note: ${question.improvement_note.substring(0, 50)}... ā (cleared)`);
|
|
2018
|
+
delete question.improvement_note;
|
|
2019
|
+
}
|
|
2020
|
+
if (clearFields.has('category') && question.failure_category) {
|
|
2021
|
+
console.log(` failure_category: ${question.failure_category} ā (cleared)`);
|
|
2022
|
+
delete question.failure_category;
|
|
2023
|
+
}
|
|
2024
|
+
|
|
2025
|
+
// Handle setting new values (can be combined with clearing other fields)
|
|
2026
|
+
if (improvedQuestion) {
|
|
2027
|
+
console.log(` improved_question: ${question.improved_question || '(not set)'} ā ${improvedQuestion}`);
|
|
2028
|
+
question.improved_question = improvedQuestion;
|
|
2029
|
+
}
|
|
2030
|
+
|
|
2031
|
+
if (improvedAnswer) {
|
|
2032
|
+
console.log(` improved_answer: ${question.improved_answer || '(not set)'} ā ${improvedAnswer}`);
|
|
2033
|
+
question.improved_answer = improvedAnswer;
|
|
2034
|
+
}
|
|
2035
|
+
|
|
2036
|
+
if (improvementNote) {
|
|
2037
|
+
console.log(` improvement_note: ${question.improvement_note || '(not set)'} ā ${improvementNote}`);
|
|
2038
|
+
question.improvement_note = improvementNote;
|
|
2039
|
+
}
|
|
2040
|
+
|
|
2041
|
+
if (category) {
|
|
2042
|
+
console.log(` failure_category: ${question.failure_category || '(not set)'} ā ${category}`);
|
|
2043
|
+
question.failure_category = category;
|
|
2044
|
+
}
|
|
2045
|
+
|
|
2046
|
+
// Save dataset with 4-space indentation
|
|
2047
|
+
await writeFile(datasetPath, JSON.stringify(dataset, null, 4));
|
|
2048
|
+
|
|
2049
|
+
console.log('\nā
Dataset updated!');
|
|
2050
|
+
console.log(' Run `pnpm run sync-improved-om-qa` to sync to prepared data');
|
|
2051
|
+
}
|
|
2052
|
+
|
|
2053
|
+
// --------------------------------------------------------------------------
|
|
2054
|
+
// Check Duplicate Observations
|
|
2055
|
+
// --------------------------------------------------------------------------
|
|
2056
|
+
|
|
2057
|
+
private async checkDuplicateObservations(questionId?: string): Promise<void> {
|
|
2058
|
+
if (!questionId) {
|
|
2059
|
+
console.log('ā Please provide a question ID with -q <question-id>');
|
|
2060
|
+
return;
|
|
2061
|
+
}
|
|
2062
|
+
|
|
2063
|
+
console.log(`\nš Checking for duplicate observations in ${questionId}...\n`);
|
|
2064
|
+
|
|
2065
|
+
// Find the om.json file
|
|
2066
|
+
const configs = ['observational-memory', 'om-gpt5-mini', 'om-gpt5', 'om-gemini-3-pro', 'om-gemini-3-flash'];
|
|
2067
|
+
let omJsonPath: string | null = null;
|
|
2068
|
+
let foundConfig: string | null = null;
|
|
2069
|
+
|
|
2070
|
+
for (const config of configs) {
|
|
2071
|
+
const path = join(this.preparedDataDir, 'longmemeval_s', config, questionId, 'om.json');
|
|
2072
|
+
if (existsSync(path)) {
|
|
2073
|
+
omJsonPath = path;
|
|
2074
|
+
foundConfig = config;
|
|
2075
|
+
break;
|
|
2076
|
+
}
|
|
2077
|
+
}
|
|
2078
|
+
|
|
2079
|
+
if (!omJsonPath) {
|
|
2080
|
+
console.log(`ā Could not find om.json for question ${questionId}`);
|
|
2081
|
+
return;
|
|
2082
|
+
}
|
|
2083
|
+
|
|
2084
|
+
console.log(`š Found data in config: ${foundConfig}`);
|
|
2085
|
+
|
|
2086
|
+
// Load and parse om.json using PersistableInMemoryMemory
|
|
2087
|
+
const storage = new PersistableInMemoryMemory({ readOnly: true });
|
|
2088
|
+
await storage.hydrate(omJsonPath);
|
|
2089
|
+
|
|
2090
|
+
// Get OM records
|
|
2091
|
+
const resourceId = `resource_${questionId}`;
|
|
2092
|
+
const records = await storage.getObservationalMemoryHistory(null, resourceId);
|
|
2093
|
+
|
|
2094
|
+
if (!records || records.length === 0) {
|
|
2095
|
+
console.log('ā No observational memory records found');
|
|
2096
|
+
return;
|
|
2097
|
+
}
|
|
2098
|
+
|
|
2099
|
+
const latestRecord = records[0];
|
|
2100
|
+
const activeObservations = latestRecord.activeObservations || '';
|
|
2101
|
+
|
|
2102
|
+
// Parse thread blocks from activeObservations
|
|
2103
|
+
const threadRegex = /<thread id="([^"]+)">([\s\S]*?)<\/thread>/g;
|
|
2104
|
+
const threadBlocks: { id: string; content: string; startIndex: number }[] = [];
|
|
2105
|
+
let match;
|
|
2106
|
+
|
|
2107
|
+
while ((match = threadRegex.exec(activeObservations)) !== null) {
|
|
2108
|
+
threadBlocks.push({
|
|
2109
|
+
id: match[1],
|
|
2110
|
+
content: match[2].trim(),
|
|
2111
|
+
startIndex: match.index,
|
|
2112
|
+
});
|
|
2113
|
+
}
|
|
2114
|
+
|
|
2115
|
+
console.log(`š Found ${threadBlocks.length} total thread blocks\n`);
|
|
2116
|
+
|
|
2117
|
+
// Count occurrences of each thread ID
|
|
2118
|
+
const threadCounts: Record<string, number> = {};
|
|
2119
|
+
for (const block of threadBlocks) {
|
|
2120
|
+
threadCounts[block.id] = (threadCounts[block.id] || 0) + 1;
|
|
2121
|
+
}
|
|
2122
|
+
|
|
2123
|
+
// Find duplicates
|
|
2124
|
+
const duplicates = Object.entries(threadCounts)
|
|
2125
|
+
.filter(([_, count]) => count > 1)
|
|
2126
|
+
.sort((a, b) => b[1] - a[1]);
|
|
2127
|
+
|
|
2128
|
+
const uniqueThreads = Object.keys(threadCounts).length;
|
|
2129
|
+
|
|
2130
|
+
console.log(`š Statistics:`);
|
|
2131
|
+
console.log(` Total thread blocks: ${threadBlocks.length}`);
|
|
2132
|
+
console.log(` Unique thread IDs: ${uniqueThreads}`);
|
|
2133
|
+
console.log(` Duplicate ratio: ${(threadBlocks.length / uniqueThreads).toFixed(2)}x\n`);
|
|
2134
|
+
|
|
2135
|
+
if (duplicates.length === 0) {
|
|
2136
|
+
console.log('ā
No duplicate thread blocks found!');
|
|
2137
|
+
return;
|
|
2138
|
+
}
|
|
2139
|
+
|
|
2140
|
+
console.log(`ā ļø Found ${duplicates.length} thread IDs with duplicates:\n`);
|
|
2141
|
+
|
|
2142
|
+
for (const [threadId, count] of duplicates.slice(0, 10)) {
|
|
2143
|
+
console.log(` ${threadId}: ${count} occurrences`);
|
|
2144
|
+
|
|
2145
|
+
// Show the content of each duplicate to see if they're identical
|
|
2146
|
+
const blocks = threadBlocks.filter(b => b.id === threadId);
|
|
2147
|
+
const contents = blocks.map(b => b.content);
|
|
2148
|
+
const uniqueContents = new Set(contents);
|
|
2149
|
+
|
|
2150
|
+
if (uniqueContents.size === 1) {
|
|
2151
|
+
console.log(` āā ā ļø All ${count} blocks have IDENTICAL content`);
|
|
2152
|
+
} else {
|
|
2153
|
+
console.log(` āā ā¹ļø ${uniqueContents.size} unique content variations (may be legitimate)`);
|
|
2154
|
+
}
|
|
2155
|
+
}
|
|
2156
|
+
|
|
2157
|
+
if (duplicates.length > 10) {
|
|
2158
|
+
console.log(`\n ... and ${duplicates.length - 10} more`);
|
|
2159
|
+
}
|
|
2160
|
+
|
|
2161
|
+
// Calculate token waste estimate
|
|
2162
|
+
const avgBlockSize = activeObservations.length / threadBlocks.length;
|
|
2163
|
+
const duplicateBlocks = threadBlocks.length - uniqueThreads;
|
|
2164
|
+
const estimatedWaste = Math.round(duplicateBlocks * avgBlockSize);
|
|
2165
|
+
|
|
2166
|
+
console.log(`\nš Estimated token waste from duplicates: ~${estimatedWaste} characters`);
|
|
2167
|
+
}
|
|
2168
|
+
|
|
2169
|
+
// --------------------------------------------------------------------------
|
|
2170
|
+
// Baseline Check - Comprehensive Data Quality Check
|
|
2171
|
+
// --------------------------------------------------------------------------
|
|
2172
|
+
|
|
2173
|
+
private async runBaselineCheck(questionId: string): Promise<void> {
|
|
2174
|
+
console.log(`\nš BASELINE CHECK for ${questionId}\n`);
|
|
2175
|
+
console.log('ā'.repeat(60));
|
|
2176
|
+
|
|
2177
|
+
// 1. Find the prepared data
|
|
2178
|
+
const configs = ['observational-memory', 'om-gpt5-mini', 'om-gpt5', 'om-gemini-3-pro', 'om-gemini-3-flash'];
|
|
2179
|
+
let omJsonPath: string | null = null;
|
|
2180
|
+
let foundConfig: string | null = null;
|
|
2181
|
+
let questionDir: string | null = null;
|
|
2182
|
+
|
|
2183
|
+
for (const config of configs) {
|
|
2184
|
+
const path = join(this.preparedDataDir, 'longmemeval_s', config, questionId, 'om.json');
|
|
2185
|
+
if (existsSync(path)) {
|
|
2186
|
+
omJsonPath = path;
|
|
2187
|
+
foundConfig = config;
|
|
2188
|
+
questionDir = join(this.preparedDataDir, 'longmemeval_s', config, questionId);
|
|
2189
|
+
break;
|
|
2190
|
+
}
|
|
2191
|
+
}
|
|
2192
|
+
|
|
2193
|
+
if (!omJsonPath || !questionDir) {
|
|
2194
|
+
console.log(`ā Could not find prepared data for question ${questionId}`);
|
|
2195
|
+
return;
|
|
2196
|
+
}
|
|
2197
|
+
|
|
2198
|
+
// 2. Get file modification time (preparation date)
|
|
2199
|
+
const { stat } = await import('fs/promises');
|
|
2200
|
+
const metaPath = join(questionDir, 'meta.json');
|
|
2201
|
+
let preparedDate = 'Unknown';
|
|
2202
|
+
if (existsSync(metaPath)) {
|
|
2203
|
+
const stats = await stat(metaPath);
|
|
2204
|
+
preparedDate = stats.mtime.toLocaleString();
|
|
2205
|
+
}
|
|
2206
|
+
|
|
2207
|
+
console.log(`\nš DATA LOCATION`);
|
|
2208
|
+
console.log(` Config: ${foundConfig}`);
|
|
2209
|
+
console.log(` Path: ${questionDir}`);
|
|
2210
|
+
console.log(` Prepared: ${preparedDate}`);
|
|
2211
|
+
|
|
2212
|
+
// 3. Load storage and check for per-thread cursors (staleness)
|
|
2213
|
+
const storage = new PersistableInMemoryMemory({ readOnly: true });
|
|
2214
|
+
await storage.hydrate(omJsonPath);
|
|
2215
|
+
|
|
2216
|
+
const resourceId = `resource_${questionId}`;
|
|
2217
|
+
|
|
2218
|
+
// Get threads and check for cursors
|
|
2219
|
+
const threadsResult = await storage.listThreads({ filter: { resourceId }, perPage: false });
|
|
2220
|
+
const threads = threadsResult.threads || [];
|
|
2221
|
+
let threadsWithCursor = 0;
|
|
2222
|
+
let totalThreads = 0;
|
|
2223
|
+
|
|
2224
|
+
for (const thread of threads) {
|
|
2225
|
+
totalThreads++;
|
|
2226
|
+
const metadata = thread.metadata as Record<string, unknown> | undefined;
|
|
2227
|
+
const mastraOm = (metadata?.mastra as Record<string, unknown>)?.om as Record<string, unknown> | undefined;
|
|
2228
|
+
if (metadata?.lastObservedAt || mastraOm?.lastObservedAt) {
|
|
2229
|
+
threadsWithCursor++;
|
|
2230
|
+
}
|
|
2231
|
+
}
|
|
2232
|
+
|
|
2233
|
+
const isStale = totalThreads > 0 && threadsWithCursor === 0;
|
|
2234
|
+
|
|
2235
|
+
console.log(`\nš DATA FRESHNESS`);
|
|
2236
|
+
console.log(` Threads: ${totalThreads}`);
|
|
2237
|
+
console.log(` With per-thread cursors: ${threadsWithCursor}/${totalThreads}`);
|
|
2238
|
+
if (isStale) {
|
|
2239
|
+
console.log(` ā ļø STATUS: STALE (prepared before per-thread cursor fix)`);
|
|
2240
|
+
console.log(` ā Recommend: pnpm prepare om -v full --question-id ${questionId} --force-regenerate -y`);
|
|
2241
|
+
} else {
|
|
2242
|
+
console.log(` ā
STATUS: FRESH (has per-thread cursors)`);
|
|
2243
|
+
}
|
|
2244
|
+
|
|
2245
|
+
// 4. Check for duplicate observations
|
|
2246
|
+
const records = await storage.getObservationalMemoryHistory(null, resourceId);
|
|
2247
|
+
let duplicateRatio = 1.0;
|
|
2248
|
+
let totalBlocks = 0;
|
|
2249
|
+
let uniqueBlocks = 0;
|
|
2250
|
+
|
|
2251
|
+
if (records && records.length > 0) {
|
|
2252
|
+
const latestRecord = records[0];
|
|
2253
|
+
const activeObservations = latestRecord.activeObservations || '';
|
|
2254
|
+
|
|
2255
|
+
const threadRegex = /<thread id=\"([^\"]+)\">[\s\S]*?<\/thread>/g;
|
|
2256
|
+
const threadIds: string[] = [];
|
|
2257
|
+
let match;
|
|
2258
|
+
while ((match = threadRegex.exec(activeObservations)) !== null) {
|
|
2259
|
+
threadIds.push(match[1]);
|
|
2260
|
+
}
|
|
2261
|
+
|
|
2262
|
+
totalBlocks = threadIds.length;
|
|
2263
|
+
uniqueBlocks = new Set(threadIds).size;
|
|
2264
|
+
duplicateRatio = totalBlocks > 0 ? totalBlocks / uniqueBlocks : 1.0;
|
|
2265
|
+
}
|
|
2266
|
+
|
|
2267
|
+
console.log(`\nš DUPLICATE OBSERVATIONS`);
|
|
2268
|
+
console.log(` Total thread blocks: ${totalBlocks}`);
|
|
2269
|
+
console.log(` Unique thread IDs: ${uniqueBlocks}`);
|
|
2270
|
+
console.log(` Duplicate ratio: ${duplicateRatio.toFixed(2)}x`);
|
|
2271
|
+
if (duplicateRatio > 1.5) {
|
|
2272
|
+
console.log(` ā ļø HIGH DUPLICATION - may waste tokens`);
|
|
2273
|
+
console.log(` ā Run: pnpm investigate --check-duplicates -q ${questionId}`);
|
|
2274
|
+
} else if (duplicateRatio > 1.0) {
|
|
2275
|
+
console.log(` ā¹ļø Minor duplication (may be legitimate)`);
|
|
2276
|
+
} else {
|
|
2277
|
+
console.log(` ā
No duplicates`);
|
|
2278
|
+
}
|
|
2279
|
+
|
|
2280
|
+
// 5. Get message and observation counts
|
|
2281
|
+
const messagesResult = await storage.listMessagesByResourceId({ resourceId, perPage: false });
|
|
2282
|
+
const messageCount = messagesResult.messages.length;
|
|
2283
|
+
|
|
2284
|
+
// Count observation lines
|
|
2285
|
+
let observationLineCount = 0;
|
|
2286
|
+
if (records && records.length > 0) {
|
|
2287
|
+
const activeObservations = records[0].activeObservations || '';
|
|
2288
|
+
observationLineCount = (activeObservations.match(/^\s*\*\s+/gm) || []).length;
|
|
2289
|
+
}
|
|
2290
|
+
|
|
2291
|
+
console.log(`\nš DATA VOLUME`);
|
|
2292
|
+
console.log(` Messages stored: ${messageCount}`);
|
|
2293
|
+
console.log(` Observation lines: ${observationLineCount}`);
|
|
2294
|
+
console.log(` OM records: ${records?.length || 0}`);
|
|
2295
|
+
|
|
2296
|
+
// 6. Summary
|
|
2297
|
+
console.log(`\n${'ā'.repeat(60)}`);
|
|
2298
|
+
console.log(`š SUMMARY`);
|
|
2299
|
+
|
|
2300
|
+
const issues: string[] = [];
|
|
2301
|
+
if (isStale) issues.push('STALE DATA');
|
|
2302
|
+
if (duplicateRatio > 1.5) issues.push('HIGH DUPLICATION');
|
|
2303
|
+
|
|
2304
|
+
if (issues.length === 0) {
|
|
2305
|
+
console.log(` ā
No data quality issues detected`);
|
|
2306
|
+
} else {
|
|
2307
|
+
console.log(` ā ļø Issues found: ${issues.join(', ')}`);
|
|
2308
|
+
}
|
|
2309
|
+
console.log('');
|
|
2310
|
+
}
|
|
2311
|
+
|
|
2312
|
+
// --------------------------------------------------------------------------
|
|
2313
|
+
// Print Prepare Command for Stale Questions
|
|
2314
|
+
// --------------------------------------------------------------------------
|
|
2315
|
+
|
|
2316
|
+
private async printPrepareCommand(): Promise<void> {
|
|
2317
|
+
console.log('\nš Finding stale/partial questions in current investigation...\n');
|
|
2318
|
+
|
|
2319
|
+
// Get current investigation
|
|
2320
|
+
const investigations = await this.listInvestigations();
|
|
2321
|
+
if (investigations.length === 0) {
|
|
2322
|
+
console.log('ā No investigations found. Run: pnpm investigate <run-id>');
|
|
2323
|
+
return;
|
|
2324
|
+
}
|
|
2325
|
+
|
|
2326
|
+
// Use most recent investigation
|
|
2327
|
+
const currentInv = investigations[0];
|
|
2328
|
+
const progress = await this.loadProgress(currentInv);
|
|
2329
|
+
if (!progress) {
|
|
2330
|
+
console.log(`ā Could not load progress for ${currentInv}`);
|
|
2331
|
+
return;
|
|
2332
|
+
}
|
|
2333
|
+
|
|
2334
|
+
// Determine the base config to use for preparation
|
|
2335
|
+
let prepareConfig = progress.config;
|
|
2336
|
+
try {
|
|
2337
|
+
const { getMemoryConfig, isValidMemoryConfig } = await import('../config');
|
|
2338
|
+
if (isValidMemoryConfig(progress.config)) {
|
|
2339
|
+
const configDef = getMemoryConfig(progress.config);
|
|
2340
|
+
if (configDef?.readOnlyConfig && configDef?.baseConfig) {
|
|
2341
|
+
prepareConfig = configDef.baseConfig;
|
|
2342
|
+
}
|
|
2343
|
+
}
|
|
2344
|
+
} catch {
|
|
2345
|
+
// Use original config
|
|
2346
|
+
}
|
|
2347
|
+
|
|
2348
|
+
console.log(`š Investigation: ${currentInv}`);
|
|
2349
|
+
console.log(` Config: ${progress.config}${prepareConfig !== progress.config ? ` (uses ${prepareConfig})` : ''}`);
|
|
2350
|
+
console.log(` Dataset: ${progress.dataset}\n`);
|
|
2351
|
+
|
|
2352
|
+
// Get pending questions
|
|
2353
|
+
const pendingQuestions = Object.entries(progress.questions)
|
|
2354
|
+
.filter(([_, q]) => q.status === 'pending')
|
|
2355
|
+
.map(([id]) => id);
|
|
2356
|
+
|
|
2357
|
+
console.log(`š Pending questions: ${pendingQuestions.length}\n`);
|
|
2358
|
+
|
|
2359
|
+
// Check each pending question for staleness
|
|
2360
|
+
const staleQuestionIds: string[] = [];
|
|
2361
|
+
|
|
2362
|
+
for (const questionId of pendingQuestions) {
|
|
2363
|
+
const { omPath, config, dataset } = await this.findPreparedData(questionId);
|
|
2364
|
+
if (!omPath) continue;
|
|
2365
|
+
|
|
2366
|
+
const result = await this.checkSingleQuestion(questionId, omPath, config, dataset);
|
|
2367
|
+
|
|
2368
|
+
// Consider stale if:
|
|
2369
|
+
// 1. No threads have cursors (completely stale)
|
|
2370
|
+
// 2. Only some threads have cursors (partial)
|
|
2371
|
+
if (result.isStale || (result.threadCount > 0 && result.threadsWithCursor < result.threadCount)) {
|
|
2372
|
+
const status = result.threadsWithCursor === 0 ? 'STALE' : 'PARTIAL';
|
|
2373
|
+
console.log(` ${status}: ${questionId} (${result.reason})`);
|
|
2374
|
+
staleQuestionIds.push(questionId);
|
|
2375
|
+
}
|
|
2376
|
+
}
|
|
2377
|
+
|
|
2378
|
+
if (staleQuestionIds.length === 0) {
|
|
2379
|
+
console.log('ā
No stale or partial questions found in pending queue!');
|
|
2380
|
+
return;
|
|
2381
|
+
}
|
|
2382
|
+
|
|
2383
|
+
console.log(`\nš Found ${staleQuestionIds.length} stale/partial question(s)\n`);
|
|
2384
|
+
console.log('Run this command to re-prepare them:\n');
|
|
2385
|
+
console.log(
|
|
2386
|
+
`pnpm prepare ${prepareConfig} -v full --question-id ${staleQuestionIds.join(',')} --force-regenerate\n`,
|
|
2387
|
+
);
|
|
2388
|
+
}
|
|
2389
|
+
|
|
2390
|
+
// Deprecated - use printPrepareCommand instead
|
|
2391
|
+
private async prepareStaleQuestions(dryRun?: boolean): Promise<void> {
|
|
2392
|
+
console.log('\nā ļø --prepare-stale is deprecated. Use --print-prepare-command instead.\n');
|
|
2393
|
+
await this.printPrepareCommand();
|
|
2394
|
+
}
|
|
2395
|
+
|
|
2396
|
+
private showHelp(): void {
|
|
2397
|
+
console.log(`
|
|
2398
|
+
š Investigation Workflow
|
|
2399
|
+
|
|
2400
|
+
Usage:
|
|
2401
|
+
pnpm investigate <run-id> Setup investigation from a benchmark run
|
|
2402
|
+
pnpm investigate --status Show progress across all investigations
|
|
2403
|
+
pnpm investigate --next Open next uninvestigated question
|
|
2404
|
+
pnpm investigate --done <question-id> Mark a question as investigated
|
|
2405
|
+
pnpm investigate --sync Sync fixes to dataset
|
|
2406
|
+
|
|
2407
|
+
Investigation Utilities:
|
|
2408
|
+
pnpm investigate --inspect <question-id> Inspect question data
|
|
2409
|
+
pnpm investigate --search <keyword> -q <id> Search observations
|
|
2410
|
+
pnpm investigate --trace <keyword> -q <id> Trace info through pipeline
|
|
2411
|
+
|
|
2412
|
+
Data Quality:
|
|
2413
|
+
pnpm investigate --baseline -q <id> Comprehensive data quality check
|
|
2414
|
+
pnpm investigate --check-stale Check all data for staleness
|
|
2415
|
+
pnpm investigate --check-stale -q <id> Check specific question
|
|
2416
|
+
pnpm investigate --check-stale --stale-only List only stale questions
|
|
2417
|
+
pnpm investigate --check-duplicates -q <id> Check for duplicate thread blocks
|
|
2418
|
+
|
|
2419
|
+
Dataset Improvements:
|
|
2420
|
+
pnpm investigate --improve <id> --improve-question "..." --improve-answer "..." --improve-note "..."
|
|
2421
|
+
|
|
2422
|
+
Examples:
|
|
2423
|
+
pnpm investigate run_1234567890
|
|
2424
|
+
pnpm investigate --inspect 07741c45
|
|
2425
|
+
pnpm investigate --search "sneaker" -q 07741c45
|
|
2426
|
+
pnpm investigate --trace "shoe rack" -q 07741c45
|
|
2427
|
+
pnpm investigate --check-stale --stale-only Find questions to re-prepare
|
|
2428
|
+
|
|
2429
|
+
Options:
|
|
2430
|
+
--output <dir> Investigation output directory (default: ./investigations)
|
|
2431
|
+
--results <dir> Results directory (default: ./results)
|
|
2432
|
+
--prepared-data <dir> Prepared data directory (default: ./prepared-data)
|
|
2433
|
+
--dataset <dir> Dataset directory (default: ./data)
|
|
2434
|
+
--editor <cmd> Editor command (default: $EDITOR or 'code')
|
|
2435
|
+
`);
|
|
2436
|
+
}
|
|
2437
|
+
}
|