@mastra/longmemeval 0.0.0-error-handler-fix-20251020202607 → 0.0.0-execa-dynamic-import-20260304221256
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +847 -6
- package/LICENSE.md +15 -0
- package/docs/INVESTIGATION_WORKFLOW.md +314 -0
- package/package.json +34 -26
- package/scripts/download.ts +22 -1
- package/scripts/generate-wm-templates.ts +2 -1
- package/src/cli.ts +967 -155
- package/src/commands/__tests__/prepare.test.ts +11 -6
- package/src/commands/__tests__/run.test.ts +10 -8
- package/src/commands/clean.ts +147 -0
- package/src/commands/deterministic-ids.ts +126 -0
- package/src/commands/find-prohibited.ts +299 -0
- package/src/commands/investigate.ts +2437 -0
- package/src/commands/list-partial.ts +137 -0
- package/src/commands/obscure-thread-ids.ts +237 -0
- package/src/commands/partial-results.ts +198 -0
- package/src/commands/precompute-embeddings.ts +302 -0
- package/src/commands/prepare.ts +732 -105
- package/src/commands/reconcile.ts +383 -0
- package/src/commands/run.ts +1240 -121
- package/src/commands/sessions.ts +126 -0
- package/src/commands/sync.ts +152 -0
- package/src/commands/test-cleaned.ts +53 -0
- package/src/commands/test-incremental.ts +120 -0
- package/src/commands/test-messages.ts +95 -0
- package/src/commands/test-prohibited.ts +244 -0
- package/src/commands/test-thread.ts +136 -0
- package/src/commands/tokens.ts +625 -0
- package/src/config.ts +1253 -72
- package/src/data/types.ts +48 -6
- package/src/embeddings/cached-openai-embedding-model.ts +0 -1
- package/src/evaluation/longmemeval-metric.ts +186 -103
- package/src/processors/content-sanitizer.ts +153 -0
- package/src/processors/date-injector.ts +111 -0
- package/src/processors/index.ts +3 -0
- package/src/processors/observation-semantic-filter.ts +1142 -0
- package/src/retry-model.ts +5 -3
- package/src/storage/__tests__/benchmark-store.test.ts +18 -35
- package/src/storage/benchmark-store.ts +140 -153
- package/src/storage/benchmark-vector.ts +1 -1
- package/src/storage/index.ts +1 -0
- package/src/storage/persistable-inmemory.ts +133 -0
- package/src/test-utils/mock-model.ts +35 -0
package/src/commands/run.ts
CHANGED
|
@@ -1,28 +1,255 @@
|
|
|
1
1
|
import { Agent } from '@mastra/core/agent';
|
|
2
|
+
import { RequestContext } from '@mastra/core/request-context';
|
|
2
3
|
import { Memory } from '@mastra/memory';
|
|
3
|
-
import {
|
|
4
|
+
import { ObservationalMemory, OBSERVATIONAL_MEMORY_DEFAULTS } from '@mastra/memory/processors';
|
|
4
5
|
import { cachedOpenAI } from '../embeddings/cached-openai-provider';
|
|
5
6
|
import chalk from 'chalk';
|
|
6
7
|
import ora, { Ora } from 'ora';
|
|
7
8
|
import { join } from 'path';
|
|
8
|
-
import { readdir, readFile, mkdir, writeFile } from 'fs/promises';
|
|
9
|
-
import { existsSync } from 'fs';
|
|
9
|
+
import { readdir, readFile, mkdir, writeFile, stat, appendFile } from 'fs/promises';
|
|
10
|
+
import { appendFileSync, existsSync, writeFileSync, mkdirSync } from 'fs';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Parse duration string (e.g., "1h", "30m", "2d") to milliseconds
|
|
14
|
+
*/
|
|
15
|
+
function parseDuration(duration: string): number {
|
|
16
|
+
const match = duration.match(/^(\d+)(m|h|d)$/);
|
|
17
|
+
if (!match) {
|
|
18
|
+
throw new Error(`Invalid duration format: ${duration}. Use format like "1h", "30m", "2d"`);
|
|
19
|
+
}
|
|
20
|
+
const value = parseInt(match[1], 10);
|
|
21
|
+
const unit = match[2];
|
|
22
|
+
switch (unit) {
|
|
23
|
+
case 'm':
|
|
24
|
+
return value * 60 * 1000;
|
|
25
|
+
case 'h':
|
|
26
|
+
return value * 60 * 60 * 1000;
|
|
27
|
+
case 'd':
|
|
28
|
+
return value * 24 * 60 * 60 * 1000;
|
|
29
|
+
default:
|
|
30
|
+
throw new Error(`Unknown duration unit: ${unit}`);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
interface FailuresFile {
|
|
35
|
+
questionIds: string[];
|
|
36
|
+
runId: string;
|
|
37
|
+
config: MemoryConfigType;
|
|
38
|
+
dataset: string;
|
|
39
|
+
timestamp: string;
|
|
40
|
+
totalFailed: number;
|
|
41
|
+
totalQuestions: number;
|
|
42
|
+
}
|
|
10
43
|
|
|
11
|
-
import { BenchmarkStore, BenchmarkVectorStore } from '../storage';
|
|
44
|
+
import { BenchmarkStore, BenchmarkVectorStore, PersistableInMemoryMemory } from '../storage';
|
|
12
45
|
import { LongMemEvalMetric } from '../evaluation/longmemeval-metric';
|
|
13
46
|
import type { EvaluationResult, BenchmarkMetrics, QuestionType, MemoryConfigType, DatasetType } from '../data/types';
|
|
14
|
-
import {
|
|
15
|
-
|
|
47
|
+
import {
|
|
48
|
+
getMemoryConfig,
|
|
49
|
+
getMemoryOptions,
|
|
50
|
+
applyStratifiedSampling,
|
|
51
|
+
applyCombSampling,
|
|
52
|
+
type MemoryConfigDefinition,
|
|
53
|
+
} from '../config';
|
|
54
|
+
import { DatasetLoader } from '../data/loader';
|
|
55
|
+
import { reconcileQuestion } from './reconcile';
|
|
56
|
+
import { ObservationSemanticFilter, DateInjector } from '../processors';
|
|
57
|
+
import { fastembed } from '@mastra/fastembed';
|
|
58
|
+
|
|
59
|
+
// Rate limit configuration
|
|
60
|
+
const RATE_LIMIT_TOKEN_THRESHOLD = 50000; // Pause if remaining tokens below this
|
|
61
|
+
|
|
62
|
+
// Shared rate limiter state across all workers
|
|
63
|
+
const rateLimiter = {
|
|
64
|
+
remainingTokens: 999999,
|
|
65
|
+
remainingRequests: 999999,
|
|
66
|
+
isWaiting: false,
|
|
67
|
+
waitPromise: null as Promise<void> | null,
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Update rate limiter state from response headers
|
|
72
|
+
*/
|
|
73
|
+
function updateRateLimiterFromResponse(response: any): void {
|
|
74
|
+
const headers = response?.response?.headers;
|
|
75
|
+
if (!headers) return;
|
|
76
|
+
|
|
77
|
+
rateLimiter.remainingTokens = parseInt(headers['x-ratelimit-remaining-tokens-minute'] || '999999', 10);
|
|
78
|
+
rateLimiter.remainingRequests = parseInt(headers['x-ratelimit-remaining-requests-minute'] || '999999', 10);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Wait for rate limit to reset (shared across all workers)
|
|
83
|
+
*/
|
|
84
|
+
async function waitForRateLimit(waitSeconds: number, reason: string): Promise<void> {
|
|
85
|
+
// If already waiting, just join the existing wait
|
|
86
|
+
if (rateLimiter.isWaiting && rateLimiter.waitPromise) {
|
|
87
|
+
return rateLimiter.waitPromise;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
rateLimiter.isWaiting = true;
|
|
91
|
+
console.log(
|
|
92
|
+
chalk.yellow(`
|
|
93
|
+
⏳ ${reason}. Waiting ${waitSeconds}s...`),
|
|
94
|
+
);
|
|
95
|
+
|
|
96
|
+
rateLimiter.waitPromise = (async () => {
|
|
97
|
+
await new Promise(resolve => setTimeout(resolve, waitSeconds * 1000));
|
|
98
|
+
rateLimiter.isWaiting = false;
|
|
99
|
+
rateLimiter.remainingTokens = 999999; // Reset after wait
|
|
100
|
+
rateLimiter.remainingRequests = 999999;
|
|
101
|
+
console.log(chalk.green(`✓ Resuming after rate limit pause`));
|
|
102
|
+
})();
|
|
103
|
+
|
|
104
|
+
return rateLimiter.waitPromise;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Check rate limit before making a request
|
|
109
|
+
*/
|
|
110
|
+
async function checkRateLimitBeforeRequest(): Promise<void> {
|
|
111
|
+
// If already waiting, join the wait
|
|
112
|
+
if (rateLimiter.isWaiting && rateLimiter.waitPromise) {
|
|
113
|
+
await rateLimiter.waitPromise;
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// Check if we're approaching limits
|
|
118
|
+
if (rateLimiter.remainingTokens < RATE_LIMIT_TOKEN_THRESHOLD || rateLimiter.remainingRequests < 5) {
|
|
119
|
+
await waitForRateLimit(
|
|
120
|
+
60,
|
|
121
|
+
`Rate limit approaching (${rateLimiter.remainingTokens} tokens, ${rateLimiter.remainingRequests} requests remaining)`,
|
|
122
|
+
);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Wrapper to handle 429 errors with retry
|
|
128
|
+
*/
|
|
129
|
+
async function withRateLimitRetry<T>(
|
|
130
|
+
fn: () => Promise<T>,
|
|
131
|
+
spinner?: Ora | { updateStatus: (status: string) => void },
|
|
132
|
+
maxRetries = 3,
|
|
133
|
+
): Promise<T> {
|
|
134
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
135
|
+
// Check rate limit before request
|
|
136
|
+
await checkRateLimitBeforeRequest();
|
|
137
|
+
|
|
138
|
+
try {
|
|
139
|
+
const result = await fn();
|
|
140
|
+
// Update rate limiter from successful response
|
|
141
|
+
updateRateLimiterFromResponse(result);
|
|
142
|
+
return result;
|
|
143
|
+
} catch (error: any) {
|
|
144
|
+
// Check if it's a 429 rate limit error (may be nested in MastraError)
|
|
145
|
+
const is429 =
|
|
146
|
+
error?.statusCode === 429 ||
|
|
147
|
+
error?.cause?.statusCode === 429 ||
|
|
148
|
+
error?.message?.includes('Too Many Requests') ||
|
|
149
|
+
error?.message?.includes('Rate limit') ||
|
|
150
|
+
error?.message?.includes('rate_limit');
|
|
151
|
+
|
|
152
|
+
// Check if it's a retryable connection error (ECONNRESET, ETIMEDOUT, etc.)
|
|
153
|
+
const isConnectionError =
|
|
154
|
+
error?.message?.includes('ECONNRESET') ||
|
|
155
|
+
error?.message?.includes('ETIMEDOUT') ||
|
|
156
|
+
error?.message?.includes('ENOTFOUND') ||
|
|
157
|
+
error?.message?.includes('ECONNREFUSED') ||
|
|
158
|
+
error?.message?.includes('Cannot connect to API') ||
|
|
159
|
+
error?.isRetryable === true ||
|
|
160
|
+
error?.cause?.code === 'ECONNRESET' ||
|
|
161
|
+
error?.cause?.code === 'ETIMEDOUT';
|
|
162
|
+
|
|
163
|
+
if (isConnectionError && !is429) {
|
|
164
|
+
// Connection error - wait a shorter time and retry
|
|
165
|
+
const waitTime = 10 + attempt * 5; // 10s, 15s, 20s...
|
|
166
|
+
|
|
167
|
+
const updateStatus = (status: string) => {
|
|
168
|
+
if (spinner && 'updateStatus' in spinner) {
|
|
169
|
+
spinner.updateStatus(status);
|
|
170
|
+
} else if (spinner && 'text' in spinner) {
|
|
171
|
+
spinner.text = status;
|
|
172
|
+
}
|
|
173
|
+
};
|
|
174
|
+
updateStatus(`Connection error - waiting ${waitTime}s (attempt ${attempt + 1}/${maxRetries})...`);
|
|
175
|
+
console.error(
|
|
176
|
+
`\n🔌 Connection error (${error?.cause?.code || 'unknown'}) - waiting ${waitTime}s before retry ${attempt + 1}/${maxRetries}`,
|
|
177
|
+
);
|
|
178
|
+
|
|
179
|
+
await waitForRateLimit(waitTime, `Connection error`);
|
|
180
|
+
|
|
181
|
+
if (attempt < maxRetries) {
|
|
182
|
+
continue; // Retry
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
if (is429) {
|
|
187
|
+
// Try to extract retry-after from nested error (check various header formats)
|
|
188
|
+
const headers = error?.responseHeaders || error?.cause?.responseHeaders || {};
|
|
189
|
+
let retryAfter = 60; // Default to 60 seconds
|
|
190
|
+
|
|
191
|
+
if (headers['retry-after']) {
|
|
192
|
+
retryAfter = parseInt(headers['retry-after'], 10);
|
|
193
|
+
} else if (headers['x-ratelimit-reset-tokens']) {
|
|
194
|
+
// Parse OpenAI format like "1m2.251s" or "4ms"
|
|
195
|
+
const resetStr = headers['x-ratelimit-reset-tokens'];
|
|
196
|
+
const minutes = resetStr.match(/(\d+)m(?!\s*s)/)?.[1];
|
|
197
|
+
const seconds = resetStr.match(/(\d+\.?\d*)s/)?.[1];
|
|
198
|
+
const ms = resetStr.match(/(\d+)ms/)?.[1];
|
|
199
|
+
retryAfter =
|
|
200
|
+
parseInt(minutes || '0', 10) * 60 +
|
|
201
|
+
Math.ceil(parseFloat(seconds || '0')) +
|
|
202
|
+
Math.ceil(parseInt(ms || '0', 10) / 1000);
|
|
203
|
+
retryAfter = Math.max(retryAfter, 5); // At least 5 seconds
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// Add a small buffer to avoid hitting the limit again immediately
|
|
207
|
+
const waitTime = retryAfter + 5;
|
|
208
|
+
|
|
209
|
+
const updateStatus = (status: string) => {
|
|
210
|
+
if (spinner && 'updateStatus' in spinner) {
|
|
211
|
+
spinner.updateStatus(status);
|
|
212
|
+
} else if (spinner && 'text' in spinner) {
|
|
213
|
+
spinner.text = status;
|
|
214
|
+
}
|
|
215
|
+
};
|
|
216
|
+
updateStatus(`Rate limited (429) - waiting ${waitTime}s (attempt ${attempt + 1}/${maxRetries})...`);
|
|
217
|
+
console.error(`\n⏳ Rate limited by OpenAI - waiting ${waitTime}s before retry ${attempt + 1}/${maxRetries}`);
|
|
218
|
+
|
|
219
|
+
await waitForRateLimit(waitTime, `Rate limited (429 error)`);
|
|
220
|
+
|
|
221
|
+
if (attempt < maxRetries) {
|
|
222
|
+
continue; // Retry
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
throw error; // Re-throw if not rate limit or max retries exceeded
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
throw new Error('Max retries exceeded');
|
|
229
|
+
}
|
|
16
230
|
|
|
17
231
|
export interface RunOptions {
|
|
18
232
|
dataset: DatasetType;
|
|
19
233
|
memoryConfig: MemoryConfigType;
|
|
20
|
-
model: string;
|
|
21
234
|
preparedDataDir?: string;
|
|
22
235
|
outputDir?: string;
|
|
23
236
|
subset?: number;
|
|
237
|
+
perTypeCount?: number;
|
|
238
|
+
offset?: number;
|
|
24
239
|
concurrency?: number;
|
|
25
240
|
questionId?: string;
|
|
241
|
+
questionType?: string;
|
|
242
|
+
// Comb sampling options
|
|
243
|
+
combSampleSize?: number;
|
|
244
|
+
combOffset?: number;
|
|
245
|
+
combStartOffset?: number;
|
|
246
|
+
// Skip improved/fixed question evaluation
|
|
247
|
+
skipFixed?: boolean;
|
|
248
|
+
// Re-run from failures
|
|
249
|
+
fromFailures?: string | boolean;
|
|
250
|
+
olderThan?: string;
|
|
251
|
+
// Resume a partial run
|
|
252
|
+
resume?: string | boolean;
|
|
26
253
|
}
|
|
27
254
|
|
|
28
255
|
interface PreparedQuestionMeta {
|
|
@@ -32,56 +259,153 @@ interface PreparedQuestionMeta {
|
|
|
32
259
|
threadIds: string[];
|
|
33
260
|
memoryConfig: string;
|
|
34
261
|
question: string;
|
|
262
|
+
improvedQuestion?: string; // Clarified version for vague/ambiguous questions
|
|
263
|
+
improvedAnswer?: string; // Updated answer for the clarified question (if different)
|
|
264
|
+
improvementNote?: string; // Notes about why this question failed (for tracking investigated failures)
|
|
265
|
+
requiresRetry?: boolean; // Eval agent sometimes fails due to poor reasoning, retry once on failure
|
|
35
266
|
answer: string;
|
|
36
267
|
evidenceSessionIds?: string[];
|
|
37
268
|
questionDate?: string;
|
|
38
269
|
}
|
|
39
270
|
|
|
40
|
-
|
|
271
|
+
/**
|
|
272
|
+
* Normalize model ID to provider/model format for ModelRouterLanguageModel
|
|
273
|
+
*/
|
|
274
|
+
function normalizeModelId(modelId: string): `${string}/${string}` {
|
|
275
|
+
return (modelId.includes('/') ? modelId : `openai/${modelId}`) as `${string}/${string}`;
|
|
276
|
+
}
|
|
41
277
|
|
|
42
278
|
export class RunCommand {
|
|
43
279
|
private preparedDataDir: string;
|
|
44
280
|
private outputDir: string;
|
|
281
|
+
private loader: DatasetLoader;
|
|
45
282
|
|
|
46
283
|
constructor() {
|
|
47
284
|
this.preparedDataDir = './prepared-data';
|
|
48
285
|
this.outputDir = './results';
|
|
286
|
+
this.loader = new DatasetLoader();
|
|
49
287
|
}
|
|
50
288
|
|
|
51
289
|
async run(options: RunOptions): Promise<BenchmarkMetrics> {
|
|
52
|
-
|
|
53
|
-
|
|
290
|
+
// Use existing run ID for resume, or generate new one
|
|
291
|
+
let runId = options.resume;
|
|
292
|
+
|
|
293
|
+
// If --resume is passed without a value (true) or empty string, find the most recent run
|
|
294
|
+
if (options.resume === true || options.resume === '') {
|
|
295
|
+
const configDir = join(options.outputDir || this.outputDir, options.memoryConfig);
|
|
296
|
+
if (existsSync(configDir)) {
|
|
297
|
+
const runs = (await readdir(configDir))
|
|
298
|
+
.filter(d => d.startsWith('run_'))
|
|
299
|
+
.sort()
|
|
300
|
+
.reverse(); // Most recent first (timestamps sort correctly)
|
|
301
|
+
if (runs.length > 0) {
|
|
302
|
+
runId = runs[0];
|
|
303
|
+
console.log(chalk.yellow(`\n🔍 Auto-detected most recent run: ${runId}\n`));
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
// Fall back to new run ID if no resume target found
|
|
309
|
+
if (!runId || runId === true) {
|
|
310
|
+
runId = `run_${Date.now()}`;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
const runDir = join(options.outputDir || this.outputDir, options.memoryConfig, runId as string);
|
|
54
314
|
await mkdir(runDir, { recursive: true });
|
|
55
315
|
|
|
56
|
-
|
|
316
|
+
// Load existing results if resuming
|
|
317
|
+
let existingResults: EvaluationResult[] = [];
|
|
318
|
+
const resultsPath = join(runDir, 'results.jsonl');
|
|
319
|
+
if (options.resume && existsSync(resultsPath)) {
|
|
320
|
+
const content = await readFile(resultsPath, 'utf-8');
|
|
321
|
+
existingResults = content
|
|
322
|
+
.trim()
|
|
323
|
+
.split('\n')
|
|
324
|
+
.filter(line => line.trim())
|
|
325
|
+
.map(line => JSON.parse(line) as EvaluationResult);
|
|
326
|
+
console.log(chalk.yellow(`\n♻️ Resuming run ${runId} with ${existingResults.length} existing results\n`));
|
|
327
|
+
}
|
|
328
|
+
const completedQuestionIds = new Set(existingResults.map(r => r.question_id));
|
|
329
|
+
|
|
330
|
+
console.log(
|
|
331
|
+
chalk.blue(`
|
|
332
|
+
🚀 ${options.resume ? 'Resuming' : 'Starting'} LongMemEval benchmark run: ${runId}
|
|
333
|
+
`),
|
|
334
|
+
);
|
|
335
|
+
const configDef = getMemoryConfig(options.memoryConfig);
|
|
336
|
+
|
|
57
337
|
console.log(chalk.gray(`Dataset: ${options.dataset}`));
|
|
58
|
-
console.log(chalk.gray(`Model: ${
|
|
338
|
+
console.log(chalk.gray(`Model: ${configDef.agentModel}`));
|
|
59
339
|
console.log(chalk.gray(`Memory Config: ${options.memoryConfig}`));
|
|
60
340
|
if (options.subset) {
|
|
61
341
|
console.log(chalk.gray(`Subset: ${options.subset} questions`));
|
|
62
342
|
}
|
|
63
343
|
console.log();
|
|
64
344
|
|
|
65
|
-
|
|
345
|
+
// For readOnlyConfig, use the base config's prepared data directly
|
|
346
|
+
const effectiveConfig =
|
|
347
|
+
configDef.readOnlyConfig && configDef.baseConfig ? configDef.baseConfig : options.memoryConfig;
|
|
348
|
+
const preparedDir = join(options.preparedDataDir || this.preparedDataDir, options.dataset, effectiveConfig);
|
|
349
|
+
|
|
350
|
+
// For readOnlyConfig, create a separate output directory for debug files
|
|
351
|
+
// This prevents writing to the base config's prepared data
|
|
352
|
+
const outputDir = configDef.readOnlyConfig
|
|
353
|
+
? join(options.preparedDataDir || this.preparedDataDir, options.dataset, options.memoryConfig)
|
|
354
|
+
: preparedDir;
|
|
66
355
|
|
|
67
356
|
if (!existsSync(preparedDir)) {
|
|
68
|
-
throw new Error(`Prepared data not found at: ${preparedDir}
|
|
357
|
+
throw new Error(`Prepared data not found at: ${preparedDir}
|
|
358
|
+
Please run 'longmemeval prepare' first.`);
|
|
69
359
|
}
|
|
70
360
|
|
|
71
|
-
|
|
361
|
+
if (configDef.readOnlyConfig && configDef.baseConfig) {
|
|
362
|
+
console.log(chalk.gray(`Using prepared data from: ${configDef.baseConfig}`));
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
// Load original dataset to get correct question order
|
|
72
366
|
const spinner = ora('Loading prepared data...').start();
|
|
367
|
+
const originalQuestions = await this.loader.loadDataset(options.dataset);
|
|
368
|
+
const questionIdOrder = new Map(originalQuestions.map((q, i) => [q.question_id, i]));
|
|
369
|
+
|
|
370
|
+
// Load prepared questions
|
|
73
371
|
const questionDirs = await readdir(preparedDir);
|
|
74
372
|
const preparedQuestions: PreparedQuestionMeta[] = [];
|
|
75
373
|
|
|
76
374
|
let skippedCount = 0;
|
|
77
375
|
let failedCount = 0;
|
|
376
|
+
let reconciledCount = 0;
|
|
78
377
|
for (const questionDir of questionDirs) {
|
|
79
378
|
const questionPath = join(preparedDir, questionDir);
|
|
80
379
|
const metaPath = join(questionPath, 'meta.json');
|
|
81
380
|
const progressPath = join(questionPath, 'progress.json');
|
|
82
381
|
|
|
83
382
|
// Check if question has been prepared
|
|
84
|
-
|
|
383
|
+
let hasMetaJson = existsSync(metaPath);
|
|
384
|
+
|
|
385
|
+
// If no meta.json and this is a derived config (non-readOnly), try to reconcile from base
|
|
386
|
+
if (!hasMetaJson && configDef.baseConfig && !configDef.readOnlyConfig) {
|
|
387
|
+
const baseDir = join(options.preparedDataDir || this.preparedDataDir, options.dataset, configDef.baseConfig);
|
|
388
|
+
const baseQuestionDir = join(baseDir, questionDir);
|
|
389
|
+
const baseMetaPath = join(baseQuestionDir, 'meta.json');
|
|
390
|
+
|
|
391
|
+
if (existsSync(baseMetaPath)) {
|
|
392
|
+
spinner.text = `Reconciling ${questionDir} from ${configDef.baseConfig}...`;
|
|
393
|
+
const result = await reconcileQuestion({
|
|
394
|
+
questionId: questionDir,
|
|
395
|
+
targetConfig: options.memoryConfig,
|
|
396
|
+
preparedDataDir: options.preparedDataDir || this.preparedDataDir,
|
|
397
|
+
dataset: options.dataset,
|
|
398
|
+
model: configDef.omModel ?? undefined,
|
|
399
|
+
});
|
|
400
|
+
|
|
401
|
+
if (result.copied) {
|
|
402
|
+
reconciledCount++;
|
|
403
|
+
hasMetaJson = true;
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
if (hasMetaJson) {
|
|
85
409
|
// Check if there's an incomplete or failed preparation
|
|
86
410
|
if (existsSync(progressPath)) {
|
|
87
411
|
const progress = JSON.parse(await readFile(progressPath, 'utf-8'));
|
|
@@ -100,48 +424,251 @@ export class RunCommand {
|
|
|
100
424
|
}
|
|
101
425
|
}
|
|
102
426
|
|
|
427
|
+
// Sort prepared questions to match original dataset order
|
|
428
|
+
preparedQuestions.sort((a, b) => {
|
|
429
|
+
const orderA = questionIdOrder.get(a.questionId) ?? Infinity;
|
|
430
|
+
const orderB = questionIdOrder.get(b.questionId) ?? Infinity;
|
|
431
|
+
return orderA - orderB;
|
|
432
|
+
});
|
|
433
|
+
|
|
434
|
+
const statusParts: string[] = [];
|
|
435
|
+
if (reconciledCount > 0) statusParts.push(`${reconciledCount} reconciled`);
|
|
436
|
+
if (skippedCount > 0) statusParts.push(`${skippedCount} incomplete`);
|
|
437
|
+
if (failedCount > 0) statusParts.push(`${failedCount} failed`);
|
|
438
|
+
|
|
103
439
|
spinner.succeed(
|
|
104
|
-
`Loaded ${preparedQuestions.length} prepared questions${
|
|
440
|
+
`Loaded ${preparedQuestions.length} prepared questions${statusParts.length > 0 ? ` (${statusParts.join(', ')})` : ''}`,
|
|
105
441
|
);
|
|
106
442
|
|
|
107
443
|
if (skippedCount > 0) {
|
|
108
444
|
console.log(
|
|
109
445
|
chalk.yellow(
|
|
110
|
-
|
|
446
|
+
`
|
|
447
|
+
⚠️ ${skippedCount} question${skippedCount > 1 ? 's' : ''} skipped due to incomplete preparation.`,
|
|
111
448
|
),
|
|
112
449
|
);
|
|
113
|
-
console.log(
|
|
450
|
+
console.log(
|
|
451
|
+
chalk.gray(` Run 'prepare' command to complete preparation.
|
|
452
|
+
`),
|
|
453
|
+
);
|
|
114
454
|
}
|
|
115
455
|
|
|
116
456
|
if (failedCount > 0) {
|
|
117
457
|
console.log(
|
|
118
|
-
chalk.red(
|
|
458
|
+
chalk.red(`
|
|
459
|
+
⚠️ ${failedCount} question${failedCount > 1 ? 's' : ''} skipped due to failed preparation.`),
|
|
460
|
+
);
|
|
461
|
+
console.log(
|
|
462
|
+
chalk.gray(` Check error logs and re-run 'prepare' command.
|
|
463
|
+
`),
|
|
119
464
|
);
|
|
120
|
-
console.log(chalk.gray(` Check error logs and re-run 'prepare' command.\n`));
|
|
121
465
|
}
|
|
122
466
|
|
|
123
467
|
// Filter by questionId if specified
|
|
124
468
|
let questionsToProcess = preparedQuestions;
|
|
125
|
-
|
|
469
|
+
|
|
470
|
+
// Handle --from-failures option
|
|
471
|
+
if (options.fromFailures) {
|
|
472
|
+
let failuresPath = typeof options.fromFailures === 'string' ? options.fromFailures : '';
|
|
473
|
+
|
|
474
|
+
// Handle flag with no value (true) or "latest" - find the most recent failures.json for this config
|
|
475
|
+
if (options.fromFailures === true || options.fromFailures === 'latest') {
|
|
476
|
+
const resultsDir = join(options.outputDir || this.outputDir, options.memoryConfig);
|
|
477
|
+
if (!existsSync(resultsDir)) {
|
|
478
|
+
throw new Error(`No results directory found for config: ${options.memoryConfig}`);
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
const runDirs = (await readdir(resultsDir))
|
|
482
|
+
.filter(d => d.startsWith('run_'))
|
|
483
|
+
.sort()
|
|
484
|
+
.reverse(); // Most recent first
|
|
485
|
+
|
|
486
|
+
let latestFailuresPath: string | null = null;
|
|
487
|
+
for (const runDir of runDirs) {
|
|
488
|
+
const candidatePath = join(resultsDir, runDir, 'failures.json');
|
|
489
|
+
if (existsSync(candidatePath)) {
|
|
490
|
+
latestFailuresPath = candidatePath;
|
|
491
|
+
break;
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
if (!latestFailuresPath) {
|
|
496
|
+
throw new Error(`No failures.json found in any run for config: ${options.memoryConfig}`);
|
|
497
|
+
}
|
|
498
|
+
failuresPath = latestFailuresPath;
|
|
499
|
+
console.log(chalk.gray(`Found latest failures: ${failuresPath}\n`));
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
// Load failures.json and filter to those question IDs
|
|
503
|
+
if (!existsSync(failuresPath)) {
|
|
504
|
+
throw new Error(`Failures file not found: ${failuresPath}`);
|
|
505
|
+
}
|
|
506
|
+
const fromFailuresData = JSON.parse(await readFile(failuresPath, 'utf-8')) as FailuresFile;
|
|
507
|
+
const failedIds = new Set(fromFailuresData.questionIds);
|
|
508
|
+
questionsToProcess = preparedQuestions.filter(q => failedIds.has(q.questionId));
|
|
509
|
+
|
|
510
|
+
if (questionsToProcess.length === 0) {
|
|
511
|
+
throw new Error(`No matching questions found for ${fromFailuresData.questionIds.length} failed IDs`);
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
// Filter by --older-than if specified (check meta.json mtime)
|
|
515
|
+
let skippedRecentCount = 0;
|
|
516
|
+
if (options.olderThan) {
|
|
517
|
+
const maxAgeMs = parseDuration(options.olderThan);
|
|
518
|
+
const cutoffTime = Date.now() - maxAgeMs;
|
|
519
|
+
|
|
520
|
+
const filteredQuestions: PreparedQuestionMeta[] = [];
|
|
521
|
+
for (const q of questionsToProcess) {
|
|
522
|
+
const metaPath = join(preparedDir, q.questionId, 'meta.json');
|
|
523
|
+
if (existsSync(metaPath)) {
|
|
524
|
+
const metaStat = await stat(metaPath);
|
|
525
|
+
if (metaStat.mtimeMs > cutoffTime) {
|
|
526
|
+
// Recently prepared, skip
|
|
527
|
+
skippedRecentCount++;
|
|
528
|
+
continue;
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
filteredQuestions.push(q);
|
|
532
|
+
}
|
|
533
|
+
questionsToProcess = filteredQuestions;
|
|
534
|
+
|
|
535
|
+
if (questionsToProcess.length === 0) {
|
|
536
|
+
console.log(
|
|
537
|
+
chalk.green(
|
|
538
|
+
`\n✓ All ${skippedRecentCount} failed questions were recently prepared (within ${options.olderThan})`,
|
|
539
|
+
),
|
|
540
|
+
);
|
|
541
|
+
console.log(chalk.gray(` Nothing to re-run.\n`));
|
|
542
|
+
// Return empty metrics
|
|
543
|
+
return {
|
|
544
|
+
total_questions: 0,
|
|
545
|
+
correct_answers: 0,
|
|
546
|
+
overall_accuracy: 0,
|
|
547
|
+
accuracy_by_type: {},
|
|
548
|
+
abstention_accuracy: 0,
|
|
549
|
+
};
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
console.log(
|
|
554
|
+
chalk.yellow(`\n🔄 Running ${questionsToProcess.length} failed questions from: ${options.fromFailures}`),
|
|
555
|
+
);
|
|
556
|
+
console.log(chalk.gray(` Run ID: ${fromFailuresData.runId}`));
|
|
557
|
+
console.log(
|
|
558
|
+
chalk.gray(` Original failures: ${fromFailuresData.totalFailed}/${fromFailuresData.totalQuestions}`),
|
|
559
|
+
);
|
|
560
|
+
if (skippedRecentCount > 0) {
|
|
561
|
+
console.log(chalk.gray(` Skipped ${skippedRecentCount} recently prepared (within ${options.olderThan})`));
|
|
562
|
+
}
|
|
563
|
+
console.log();
|
|
564
|
+
} else if (options.questionId) {
|
|
126
565
|
questionsToProcess = preparedQuestions.filter(q => q.questionId === options.questionId);
|
|
127
566
|
if (questionsToProcess.length === 0) {
|
|
128
567
|
throw new Error(`Question with ID "${options.questionId}" not found in prepared data`);
|
|
129
568
|
}
|
|
130
|
-
console.log(chalk.yellow(`\nFocusing on question: ${options.questionId}\n`));
|
|
131
|
-
} else if (options.subset) {
|
|
132
|
-
// Apply subset if requested
|
|
133
|
-
questionsToProcess = preparedQuestions.slice(0, options.subset);
|
|
134
569
|
console.log(
|
|
135
|
-
chalk.
|
|
570
|
+
chalk.yellow(`
|
|
571
|
+
Focusing on question: ${options.questionId}
|
|
572
|
+
`),
|
|
136
573
|
);
|
|
574
|
+
} else {
|
|
575
|
+
// Filter by question type(s) if specified (supports comma-separated)
|
|
576
|
+
if (options.questionType) {
|
|
577
|
+
const requestedTypes = options.questionType.split(',').map(t => t.trim());
|
|
578
|
+
const availableTypes = [...new Set(preparedQuestions.map(q => q.questionType))].sort();
|
|
579
|
+
|
|
580
|
+
// Validate all requested types exist
|
|
581
|
+
const invalidTypes = requestedTypes.filter(t => !availableTypes.includes(t));
|
|
582
|
+
if (invalidTypes.length > 0) {
|
|
583
|
+
throw new Error(
|
|
584
|
+
`Invalid question type(s): ${invalidTypes.join(', ')}. Available types: ${availableTypes.join(', ')}`,
|
|
585
|
+
);
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
questionsToProcess = preparedQuestions.filter(q => requestedTypes.includes(q.questionType));
|
|
589
|
+
if (questionsToProcess.length === 0) {
|
|
590
|
+
throw new Error(
|
|
591
|
+
`No questions found with type(s) "${options.questionType}". Available types: ${availableTypes.join(', ')}`,
|
|
592
|
+
);
|
|
593
|
+
}
|
|
594
|
+
console.log(
|
|
595
|
+
chalk.yellow(`
|
|
596
|
+
Filtering to question type${requestedTypes.length > 1 ? 's' : ''}: ${requestedTypes.join(', ')} (${questionsToProcess.length} questions)
|
|
597
|
+
`),
|
|
598
|
+
);
|
|
599
|
+
}
|
|
600
|
+
// Apply stratified sampling if perTypeCount is set (only if not filtering by type)
|
|
601
|
+
if (options.perTypeCount && !options.questionType) {
|
|
602
|
+
console.log(
|
|
603
|
+
chalk.gray(`
|
|
604
|
+
Applying stratified sampling (${options.perTypeCount} per type):`),
|
|
605
|
+
);
|
|
606
|
+
// Map prepared questions to include question_type for sampling
|
|
607
|
+
const withTypes = preparedQuestions.map(q => ({
|
|
608
|
+
...q,
|
|
609
|
+
question_id: q.questionId,
|
|
610
|
+
question_type: q.questionType,
|
|
611
|
+
}));
|
|
612
|
+
const sampled = applyStratifiedSampling(withTypes, options.perTypeCount);
|
|
613
|
+
questionsToProcess = sampled.map(q => {
|
|
614
|
+
const { question_id, question_type, ...rest } = q;
|
|
615
|
+
return rest as (typeof preparedQuestions)[0];
|
|
616
|
+
});
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
// Apply comb sampling if combSampleSize is set (works with -t type filter)
|
|
620
|
+
if (options.combSampleSize) {
|
|
621
|
+
const combOffset = options.combOffset ?? 10;
|
|
622
|
+
const startOffset = options.combStartOffset ?? 0;
|
|
623
|
+
console.log(
|
|
624
|
+
chalk.gray(`
|
|
625
|
+
Applying comb sampling (${options.combSampleSize} per type, offset=${combOffset}, start=${startOffset}):`),
|
|
626
|
+
);
|
|
627
|
+
// Map prepared questions to include question_type for sampling
|
|
628
|
+
const withTypes = questionsToProcess.map(q => ({
|
|
629
|
+
...q,
|
|
630
|
+
question_id: q.questionId,
|
|
631
|
+
question_type: q.questionType,
|
|
632
|
+
}));
|
|
633
|
+
const sampled = applyCombSampling(withTypes, options.combSampleSize, combOffset, startOffset);
|
|
634
|
+
questionsToProcess = sampled.map(q => {
|
|
635
|
+
const { question_id, question_type, ...rest } = q;
|
|
636
|
+
return rest as (typeof preparedQuestions)[0];
|
|
637
|
+
});
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
// Apply offset and subset
|
|
641
|
+
const totalBeforeSlice = questionsToProcess.length;
|
|
642
|
+
const offset = options.offset || 0;
|
|
643
|
+
if (offset > 0) {
|
|
644
|
+
questionsToProcess = questionsToProcess.slice(offset);
|
|
645
|
+
}
|
|
646
|
+
if (options.subset) {
|
|
647
|
+
questionsToProcess = questionsToProcess.slice(0, options.subset);
|
|
648
|
+
}
|
|
649
|
+
if (offset > 0 || options.subset) {
|
|
650
|
+
console.log(
|
|
651
|
+
chalk.gray(`Processing questions ${offset + 1}-${offset + questionsToProcess.length} of ${totalBeforeSlice}${options.questionType ? ` ${options.questionType}` : ''} total
|
|
652
|
+
`),
|
|
653
|
+
);
|
|
654
|
+
}
|
|
137
655
|
}
|
|
138
656
|
|
|
139
657
|
console.log(
|
|
140
|
-
chalk.yellow(
|
|
658
|
+
chalk.yellow(`
|
|
659
|
+
Evaluating ${questionsToProcess.length} question${questionsToProcess.length !== 1 ? 's' : ''}
|
|
660
|
+
`),
|
|
141
661
|
);
|
|
142
662
|
|
|
663
|
+
// Filter out already-completed questions (for resume)
|
|
664
|
+
const remainingQuestions = questionsToProcess.filter(q => !completedQuestionIds.has(q.questionId));
|
|
665
|
+
if (options.resume && remainingQuestions.length < questionsToProcess.length) {
|
|
666
|
+
console.log(
|
|
667
|
+
chalk.yellow(`Skipping ${questionsToProcess.length - remainingQuestions.length} already-completed questions`),
|
|
668
|
+
);
|
|
669
|
+
}
|
|
670
|
+
|
|
143
671
|
// Process questions with concurrency control
|
|
144
|
-
const results: EvaluationResult[] = [];
|
|
145
672
|
const concurrency = options.concurrency || 5;
|
|
146
673
|
const questionSpinner = ora('Evaluating questions...').start();
|
|
147
674
|
|
|
@@ -153,16 +680,19 @@ export class RunCommand {
|
|
|
153
680
|
const activeEvaluations = new Map<number, { questionId: string; status: string }>();
|
|
154
681
|
|
|
155
682
|
// Function to update progress display
|
|
683
|
+
const totalToProcess = remainingQuestions.length;
|
|
156
684
|
let lastText = '';
|
|
157
685
|
const updateProgress = () => {
|
|
158
686
|
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
|
159
687
|
const rate = elapsed > 0 ? completedCount / elapsed : 0;
|
|
160
|
-
const remaining = rate > 0 ? Math.round((
|
|
688
|
+
const remaining = rate > 0 ? Math.round((totalToProcess - completedCount) / rate) : 0;
|
|
161
689
|
|
|
162
|
-
let progressText = `Overall: ${completedCount}/${
|
|
690
|
+
let progressText = `Overall: ${completedCount}/${totalToProcess} (${inProgressCount} in progress, ${Math.round(rate * 60)} q/min, ~${remaining}s remaining)`;
|
|
163
691
|
|
|
164
692
|
if (activeEvaluations.size > 0 && concurrency > 1) {
|
|
165
|
-
progressText +=
|
|
693
|
+
progressText += `
|
|
694
|
+
|
|
695
|
+
Active evaluations:`;
|
|
166
696
|
|
|
167
697
|
// Sort active evaluations by completion status
|
|
168
698
|
const sortedEvaluations = Array.from(activeEvaluations.entries())
|
|
@@ -180,7 +710,8 @@ export class RunCommand {
|
|
|
180
710
|
|
|
181
711
|
sortedEvaluations.forEach(({ index, info, progress }) => {
|
|
182
712
|
const percentage = (progress * 100).toFixed(0);
|
|
183
|
-
progressText +=
|
|
713
|
+
progressText += `
|
|
714
|
+
[${index + 1}] ${info.questionId} - ${info.status} (${percentage}%)`;
|
|
184
715
|
});
|
|
185
716
|
}
|
|
186
717
|
|
|
@@ -191,12 +722,10 @@ export class RunCommand {
|
|
|
191
722
|
};
|
|
192
723
|
|
|
193
724
|
// Create a queue of questions to evaluate
|
|
194
|
-
const questionQueue = [...
|
|
725
|
+
const questionQueue = [...remainingQuestions];
|
|
195
726
|
|
|
196
727
|
// Function to process next question from queue
|
|
197
|
-
const processNextQuestion = async (slotIndex: number): Promise<
|
|
198
|
-
const workerResults: EvaluationResult[] = [];
|
|
199
|
-
|
|
728
|
+
const processNextQuestion = async (slotIndex: number): Promise<void> => {
|
|
200
729
|
while (questionQueue.length > 0) {
|
|
201
730
|
const meta = questionQueue.shift();
|
|
202
731
|
if (!meta) break;
|
|
@@ -208,8 +737,10 @@ export class RunCommand {
|
|
|
208
737
|
const result = await this.evaluateQuestion(
|
|
209
738
|
meta,
|
|
210
739
|
preparedDir,
|
|
211
|
-
|
|
740
|
+
outputDir,
|
|
741
|
+
normalizeModelId(configDef.agentModel ?? 'gpt-4o'),
|
|
212
742
|
options,
|
|
743
|
+
configDef,
|
|
213
744
|
concurrency > 1
|
|
214
745
|
? {
|
|
215
746
|
updateStatus: (status: string) => {
|
|
@@ -240,15 +771,25 @@ export class RunCommand {
|
|
|
240
771
|
console.log(chalk.yellow(` Expected: "${meta.answer}"`));
|
|
241
772
|
}
|
|
242
773
|
|
|
774
|
+
// Show improved result if applicable
|
|
775
|
+
if (result.improved_question) {
|
|
776
|
+
console.log(
|
|
777
|
+
chalk.cyan(` ↳ improved:`),
|
|
778
|
+
chalk[result.improved_is_correct ? 'green' : 'red'](`${result.improved_is_correct ? '✓' : '✗'}`),
|
|
779
|
+
);
|
|
780
|
+
// if (!result.improved_is_correct) {
|
|
781
|
+
console.log(chalk.gray(` Q: "${result.improved_question}"`));
|
|
782
|
+
console.log(chalk.gray(` A: "${result.improved_hypothesis}"`));
|
|
783
|
+
// }
|
|
784
|
+
}
|
|
785
|
+
|
|
243
786
|
// Re-render the spinner
|
|
244
787
|
questionSpinner.render();
|
|
245
788
|
}
|
|
246
789
|
|
|
247
|
-
//
|
|
248
|
-
|
|
790
|
+
// Append result to JSONL file immediately (crash-safe incremental save)
|
|
791
|
+
await appendFile(resultsPath, JSON.stringify(result) + '\n');
|
|
249
792
|
}
|
|
250
|
-
|
|
251
|
-
return workerResults;
|
|
252
793
|
};
|
|
253
794
|
|
|
254
795
|
// Set up periodic progress updates
|
|
@@ -257,28 +798,38 @@ export class RunCommand {
|
|
|
257
798
|
// Create worker slots
|
|
258
799
|
const workers = Array.from({ length: concurrency }, (_, i) => processNextQuestion(i));
|
|
259
800
|
|
|
260
|
-
// Wait for all workers to complete
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
// Process results from all workers
|
|
264
|
-
for (const workerResultArray of workerResults) {
|
|
265
|
-
results.push(...workerResultArray);
|
|
266
|
-
}
|
|
801
|
+
// Wait for all workers to complete
|
|
802
|
+
await Promise.all(workers);
|
|
267
803
|
|
|
268
804
|
// Clear the interval
|
|
269
805
|
clearInterval(progressInterval);
|
|
270
806
|
|
|
271
|
-
questionSpinner.succeed(`Evaluated ${
|
|
807
|
+
questionSpinner.succeed(`Evaluated ${totalToProcess} questions`);
|
|
808
|
+
|
|
809
|
+
// Load all results from JSONL file (includes existing + new)
|
|
810
|
+
const allResultsContent = await readFile(resultsPath, 'utf-8');
|
|
811
|
+
const allResults = allResultsContent
|
|
812
|
+
.trim()
|
|
813
|
+
.split('\n')
|
|
814
|
+
.filter(line => line.trim())
|
|
815
|
+
.map(line => JSON.parse(line) as EvaluationResult);
|
|
272
816
|
|
|
273
817
|
// Calculate metrics
|
|
274
|
-
console.log(
|
|
275
|
-
|
|
818
|
+
console.log(
|
|
819
|
+
chalk.blue(`
|
|
820
|
+
📊 Calculating metrics...
|
|
821
|
+
`),
|
|
822
|
+
);
|
|
823
|
+
const metrics = this.calculateMetrics(allResults);
|
|
276
824
|
|
|
277
|
-
// Save results
|
|
278
|
-
await this.
|
|
825
|
+
// Save final metrics and failures (results.jsonl already saved incrementally)
|
|
826
|
+
await this.saveFinalResults(runDir, allResults, metrics, options);
|
|
279
827
|
|
|
280
|
-
// Display
|
|
281
|
-
this.
|
|
828
|
+
// Display uninvestigated failures first (questions for investigation)
|
|
829
|
+
this.displayUninvestigatedFailures(allResults);
|
|
830
|
+
|
|
831
|
+
// Display results summary at the end
|
|
832
|
+
this.displayMetrics(metrics, options, configDef);
|
|
282
833
|
|
|
283
834
|
return metrics;
|
|
284
835
|
}
|
|
@@ -286,8 +837,10 @@ export class RunCommand {
|
|
|
286
837
|
private async evaluateQuestion(
|
|
287
838
|
meta: PreparedQuestionMeta,
|
|
288
839
|
preparedDir: string,
|
|
289
|
-
|
|
840
|
+
outputDir: string,
|
|
841
|
+
agentModelId: `${string}/${string}`,
|
|
290
842
|
options: RunOptions,
|
|
843
|
+
configDef: MemoryConfigDefinition,
|
|
291
844
|
spinner?: Ora | { updateStatus: (status: string) => void },
|
|
292
845
|
): Promise<EvaluationResult> {
|
|
293
846
|
const questionStart = Date.now();
|
|
@@ -305,11 +858,20 @@ export class RunCommand {
|
|
|
305
858
|
|
|
306
859
|
// Load the prepared storage and vector store
|
|
307
860
|
const questionDir = join(preparedDir, meta.questionId);
|
|
308
|
-
|
|
861
|
+
// Separate output directory for debug files (prevents writing to base config for readOnlyConfig)
|
|
862
|
+
const questionOutputDir = join(outputDir, meta.questionId);
|
|
309
863
|
const benchmarkVectorStore = new BenchmarkVectorStore('read');
|
|
310
864
|
|
|
311
|
-
|
|
312
|
-
|
|
865
|
+
const memoryOptions = getMemoryOptions(options.memoryConfig);
|
|
866
|
+
const usesObservationalMemory = configDef.usesObservationalMemory;
|
|
867
|
+
|
|
868
|
+
// Only load BenchmarkStore for non-OM configs (OM uses PersistableInMemoryMemory)
|
|
869
|
+
let benchmarkStore: BenchmarkStore | undefined;
|
|
870
|
+
if (!usesObservationalMemory) {
|
|
871
|
+
benchmarkStore = new BenchmarkStore('read');
|
|
872
|
+
await benchmarkStore.init();
|
|
873
|
+
await benchmarkStore.hydrate(join(questionDir, 'db.json'));
|
|
874
|
+
}
|
|
313
875
|
|
|
314
876
|
// Hydrate vector store if it exists
|
|
315
877
|
const vectorPath = join(questionDir, 'vector.json');
|
|
@@ -318,45 +880,237 @@ export class RunCommand {
|
|
|
318
880
|
updateStatus(`Loading vector embeddings for ${meta.questionId}...`);
|
|
319
881
|
}
|
|
320
882
|
|
|
321
|
-
|
|
883
|
+
// Create memory with the hydrated stores (for non-OM configs)
|
|
884
|
+
// Note: BenchmarkStore is outdated and doesn't fully implement MastraStorage
|
|
885
|
+
// Using 'as any' as a workaround since OM configs use PersistableInMemoryMemory instead
|
|
886
|
+
const memory = usesObservationalMemory
|
|
887
|
+
? undefined
|
|
888
|
+
: new Memory({
|
|
889
|
+
storage: benchmarkStore as any,
|
|
890
|
+
vector: benchmarkVectorStore,
|
|
891
|
+
embedder: cachedOpenAI.embedding('text-embedding-3-small'),
|
|
892
|
+
options: memoryOptions.options,
|
|
893
|
+
});
|
|
322
894
|
|
|
323
|
-
// Create memory
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
895
|
+
// Create observational memory processor if using OM config
|
|
896
|
+
let observationalMemory: ObservationalMemory | undefined;
|
|
897
|
+
let omStorage: PersistableInMemoryMemory | undefined;
|
|
898
|
+
|
|
899
|
+
if (usesObservationalMemory) {
|
|
900
|
+
// Use PersistableInMemoryMemory for ObservationalMemory
|
|
901
|
+
omStorage = new PersistableInMemoryMemory({ readOnly: true });
|
|
902
|
+
|
|
903
|
+
// Hydrate OM storage from prepared data
|
|
904
|
+
const omPath = join(questionDir, 'om.json');
|
|
905
|
+
if (existsSync(omPath)) {
|
|
906
|
+
await omStorage.hydrate(omPath);
|
|
907
|
+
updateStatus(`Loaded OM data for ${meta.questionId}...`);
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
observationalMemory = new ObservationalMemory({
|
|
911
|
+
obscureThreadIds: true, // can't show answer_x in context when we put the thread id in xml tags
|
|
912
|
+
storage: omStorage,
|
|
913
|
+
observation: {
|
|
914
|
+
// model: retry4o.model,
|
|
915
|
+
messageTokens: OBSERVATIONAL_MEMORY_DEFAULTS.observation.messageTokens,
|
|
916
|
+
recognizePatterns: false,
|
|
917
|
+
},
|
|
918
|
+
reflection: {
|
|
919
|
+
// model: retry4o.model,
|
|
920
|
+
observationTokens: OBSERVATIONAL_MEMORY_DEFAULTS.reflection.observationTokens,
|
|
921
|
+
recognizePatterns: false,
|
|
922
|
+
},
|
|
923
|
+
scope: 'resource',
|
|
924
|
+
});
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
// Create observation semantic filter if using RAG config
|
|
928
|
+
let observationRagFilter: ObservationSemanticFilter | undefined;
|
|
929
|
+
const usesObservationRag = configDef.usesObservationRag;
|
|
930
|
+
|
|
931
|
+
if (usesObservationRag && usesObservationalMemory) {
|
|
932
|
+
// Use a shared cache directory for embeddings (content-based, reusable across configs)
|
|
933
|
+
// Subdirectory by model name for future multi-model support
|
|
934
|
+
const embeddingsCacheDir = join(preparedDir, '..', '.embeddings-cache', 'fastembed-small');
|
|
935
|
+
observationRagFilter = new ObservationSemanticFilter({
|
|
936
|
+
embedder: fastembed.small,
|
|
937
|
+
topK: configDef.ragTopK ?? 50,
|
|
938
|
+
// minSimilarity: 0.4,
|
|
939
|
+
includeCurrentTask: false,
|
|
940
|
+
includeSuggestedResponse: false,
|
|
941
|
+
includePatterns: false,
|
|
942
|
+
cacheDir: embeddingsCacheDir,
|
|
943
|
+
preferenceBoost: configDef.ragPreferenceBoost ?? false,
|
|
944
|
+
});
|
|
945
|
+
}
|
|
330
946
|
|
|
331
947
|
// Create agent with the specified model
|
|
332
948
|
const agentInstructions = `You are a helpful assistant with access to extensive conversation history.
|
|
333
|
-
When answering questions, carefully review the conversation history to identify and use any relevant user preferences, interests, or specific details they have mentioned
|
|
334
|
-
|
|
335
|
-
|
|
949
|
+
When answering questions, carefully review the conversation history to identify and use any relevant user preferences, interests, or specific details they have mentioned.`;
|
|
950
|
+
|
|
951
|
+
// Ensure output directory exists for readOnlyConfig
|
|
952
|
+
if (!existsSync(questionOutputDir)) {
|
|
953
|
+
mkdirSync(questionOutputDir, { recursive: true });
|
|
954
|
+
}
|
|
955
|
+
const omDebugPath = join(questionOutputDir, 'om.md');
|
|
956
|
+
|
|
957
|
+
// Create date injector processor if we have a question date
|
|
958
|
+
// This injects the date into the user message in official LongMemEval format:
|
|
959
|
+
// "Current Date: {date}\nQuestion: {question}"
|
|
960
|
+
const dateInjector = meta.questionDate ? new DateInjector({ date: meta.questionDate }) : null;
|
|
336
961
|
|
|
337
962
|
const agent = new Agent({
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
963
|
+
id: 'longmemeval-agent',
|
|
964
|
+
name: 'LongMemEval Agent',
|
|
965
|
+
model: agentModelId,
|
|
966
|
+
instructions: [
|
|
967
|
+
{ role: 'system', content: agentInstructions },
|
|
968
|
+
// prevent openai prompt caching
|
|
969
|
+
{
|
|
970
|
+
role: 'system',
|
|
971
|
+
content: `
|
|
972
|
+
cache: ${Math.random()}`,
|
|
973
|
+
},
|
|
974
|
+
],
|
|
975
|
+
// tools: observationalMemory
|
|
976
|
+
// ? {
|
|
977
|
+
// recall: observationalMemory?.getRecallTool(),
|
|
978
|
+
// }
|
|
979
|
+
// : undefined,
|
|
341
980
|
memory,
|
|
981
|
+
// For OM, use processors instead of memory
|
|
982
|
+
// OM handles message loading itself via cursor-based loadUnobservedMessages
|
|
983
|
+
// MessageHistory must come first in output to save messages before OM observes them
|
|
984
|
+
inputProcessors: usesObservationalMemory
|
|
985
|
+
? [
|
|
986
|
+
observationalMemory!,
|
|
987
|
+
// Add RAG filter after OM if enabled - it will filter OM's injected observations
|
|
988
|
+
...(observationRagFilter ? [observationRagFilter] : []),
|
|
989
|
+
// Inject date into user message in official LongMemEval format
|
|
990
|
+
// Placed BEFORE RAG so the date is part of the RAG query (more authentic to benchmark)
|
|
991
|
+
// Can be moved after RAG to experiment with ordering
|
|
992
|
+
...(dateInjector ? [dateInjector] : []),
|
|
993
|
+
{
|
|
994
|
+
id: 'debug',
|
|
995
|
+
processInputStep: args => {
|
|
996
|
+
// Check tagged OM messages first, then fall back to all system messages
|
|
997
|
+
// (RAG filter replaces tagged messages with untagged ones)
|
|
998
|
+
let omm = args.messageList.getSystemMessages(`observational-memory`);
|
|
999
|
+
const taggedCount = omm.length;
|
|
1000
|
+
if (!omm.length) {
|
|
1001
|
+
// RAG filter may have replaced tagged messages - get all system messages
|
|
1002
|
+
omm = args.messageList.getAllSystemMessages();
|
|
1003
|
+
}
|
|
1004
|
+
// Find the message with observations (contains <observations> tag)
|
|
1005
|
+
const observationsMsg = omm.find(m => {
|
|
1006
|
+
const content = typeof m.content === 'string' ? m.content : '';
|
|
1007
|
+
return content.includes('<observations>');
|
|
1008
|
+
});
|
|
1009
|
+
const msgToWrite = observationsMsg || omm[0];
|
|
1010
|
+
if (msgToWrite?.content) {
|
|
1011
|
+
const content =
|
|
1012
|
+
typeof msgToWrite.content === 'string' ? msgToWrite.content : JSON.stringify(msgToWrite.content);
|
|
1013
|
+
writeFileSync(
|
|
1014
|
+
omDebugPath,
|
|
1015
|
+
`[Debug: tagged=${taggedCount}, total=${omm.length}, hasObservations=${!!observationsMsg}]
|
|
1016
|
+
|
|
1017
|
+
` +
|
|
1018
|
+
content +
|
|
1019
|
+
`
|
|
1020
|
+
|
|
1021
|
+
${JSON.stringify(args.messageList.get.all.core(), null, 2)}
|
|
1022
|
+
|
|
1023
|
+
${JSON.stringify(args.requestContext?.get('MastraMemory') || {}, null, 2)}`,
|
|
1024
|
+
);
|
|
1025
|
+
}
|
|
1026
|
+
return args.messageList;
|
|
1027
|
+
},
|
|
1028
|
+
},
|
|
1029
|
+
]
|
|
1030
|
+
: undefined,
|
|
1031
|
+
outputProcessors: usesObservationalMemory
|
|
1032
|
+
? [
|
|
1033
|
+
{
|
|
1034
|
+
id: 'debug-output',
|
|
1035
|
+
processOutputResult: args => {
|
|
1036
|
+
const responses = args.messageList.get.response.v1();
|
|
1037
|
+
if (existsSync(omDebugPath)) {
|
|
1038
|
+
appendFileSync(
|
|
1039
|
+
omDebugPath,
|
|
1040
|
+
`
|
|
1041
|
+
${JSON.stringify(responses, null, 2)}`,
|
|
1042
|
+
);
|
|
1043
|
+
}
|
|
1044
|
+
return args.messageList;
|
|
1045
|
+
},
|
|
1046
|
+
},
|
|
1047
|
+
]
|
|
1048
|
+
: undefined,
|
|
342
1049
|
});
|
|
343
1050
|
|
|
344
1051
|
// Create a fresh thread for the evaluation question
|
|
345
1052
|
const evalThreadId = `eval_${meta.questionId}_${Date.now()}`;
|
|
346
1053
|
|
|
1054
|
+
// Parse questionDate for relative time annotations in OM context
|
|
1055
|
+
// Format: "2023/05/30 (Tue) 10:18"
|
|
1056
|
+
let questionDate: Date | undefined;
|
|
1057
|
+
if (meta.questionDate) {
|
|
1058
|
+
const match = meta.questionDate.match(/^(\d{4})\/(\d{2})\/(\d{2}).*?(\d{2}):(\d{2})/);
|
|
1059
|
+
if (match) {
|
|
1060
|
+
const [, year, month, day, hour, minute] = match;
|
|
1061
|
+
questionDate = new Date(
|
|
1062
|
+
parseInt(year),
|
|
1063
|
+
parseInt(month) - 1, // JS months are 0-indexed
|
|
1064
|
+
parseInt(day),
|
|
1065
|
+
parseInt(hour),
|
|
1066
|
+
parseInt(minute),
|
|
1067
|
+
);
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
// Create request context with currentDate for OM relative time annotations
|
|
1072
|
+
// Also set MastraMemory context so ObservationalMemory processor can find threadId/resourceId
|
|
1073
|
+
const requestContext = new RequestContext([
|
|
1074
|
+
['currentDate', questionDate ?? new Date()],
|
|
1075
|
+
['MastraMemory', { thread: { id: evalThreadId }, resourceId: meta.resourceId }],
|
|
1076
|
+
]);
|
|
1077
|
+
|
|
347
1078
|
updateStatus(`${meta.threadIds.length} sessions, ${options.memoryConfig}`);
|
|
348
1079
|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
1080
|
+
let response = await withRateLimitRetry(
|
|
1081
|
+
() =>
|
|
1082
|
+
agent.generate(meta.question, {
|
|
1083
|
+
threadId: evalThreadId,
|
|
1084
|
+
resourceId: meta.resourceId,
|
|
1085
|
+
requestContext,
|
|
1086
|
+
modelSettings: {
|
|
1087
|
+
temperature: 0,
|
|
1088
|
+
},
|
|
1089
|
+
context: [],
|
|
1090
|
+
}),
|
|
1091
|
+
spinner,
|
|
1092
|
+
);
|
|
1093
|
+
|
|
1094
|
+
// Track token usage from the main question
|
|
1095
|
+
let usage = response.totalUsage ?? response.usage;
|
|
1096
|
+
|
|
1097
|
+
console.log(
|
|
1098
|
+
response.text +
|
|
1099
|
+
`
|
|
1100
|
+
|
|
1101
|
+
`,
|
|
1102
|
+
);
|
|
355
1103
|
|
|
1104
|
+
const evalModelId = normalizeModelId(configDef.evalModel ?? 'openai/gpt-4o');
|
|
356
1105
|
const evalAgent = new Agent({
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
1106
|
+
id: 'longmemeval-metric-agent',
|
|
1107
|
+
name: 'LongMemEval Metric Agent',
|
|
1108
|
+
model: evalModelId,
|
|
1109
|
+
// Official LongMemEval uses no system prompt for the judge - just the per-question-type
|
|
1110
|
+
// prompts sent as user messages. Keeping empty to match official methodology.
|
|
1111
|
+
// Old instructions (non-standard):
|
|
1112
|
+
// 'You are an evaluation assistant. Answer questions precisely and concisely. Any answer to a question you see where the answer contains "I dont know", but the answer is also stated, is correct. Give leeway for conversion units. If the answer is 1.5 hours, but the given answer is 90 minutes, that is also correct. If the response contains the answer plus additional information, that is correct too.',
|
|
1113
|
+
instructions: '',
|
|
360
1114
|
});
|
|
361
1115
|
|
|
362
1116
|
const metric = new LongMemEvalMetric({
|
|
@@ -370,17 +1124,145 @@ Be specific rather than generic when the user has expressed clear preferences in
|
|
|
370
1124
|
answer: meta.answer,
|
|
371
1125
|
});
|
|
372
1126
|
|
|
373
|
-
const result = await metric.measure(input, response.text);
|
|
374
|
-
|
|
1127
|
+
const result = await withRateLimitRetry(() => metric.measure(input, response.text), spinner);
|
|
1128
|
+
let isCorrect = result.score === 1;
|
|
1129
|
+
|
|
1130
|
+
// Check if there's an improved version - if so, we'll only retry that one
|
|
1131
|
+
const hasImprovedVersion = !!(meta.improvedQuestion || meta.improvedAnswer);
|
|
1132
|
+
|
|
1133
|
+
// Retry failed evaluations: always at least 1 retry, up to 5 if requiresRetry is set
|
|
1134
|
+
// Only retry vanilla if there's NO improved version (otherwise we retry the improved one)
|
|
1135
|
+
let retryCount = 0;
|
|
1136
|
+
const maxRetries = meta.requiresRetry || hasImprovedVersion ? 2 : 0;
|
|
1137
|
+
while (!isCorrect && !hasImprovedVersion && retryCount < maxRetries) {
|
|
1138
|
+
retryCount++;
|
|
1139
|
+
updateStatus(`Retry ${retryCount}/${maxRetries} for ${meta.questionId}...`);
|
|
1140
|
+
|
|
1141
|
+
const retryThreadId = `eval_retry_${meta.questionId}_${retryCount}_${Date.now()}`;
|
|
1142
|
+
const retryResponse = await withRateLimitRetry(
|
|
1143
|
+
() =>
|
|
1144
|
+
agent.generate(meta.question, {
|
|
1145
|
+
threadId: retryThreadId,
|
|
1146
|
+
resourceId: meta.resourceId,
|
|
1147
|
+
requestContext,
|
|
1148
|
+
modelSettings: {
|
|
1149
|
+
temperature: 0,
|
|
1150
|
+
},
|
|
1151
|
+
}),
|
|
1152
|
+
spinner,
|
|
1153
|
+
);
|
|
1154
|
+
|
|
1155
|
+
const retryResult = await withRateLimitRetry(() => metric.measure(input, retryResponse.text), spinner);
|
|
1156
|
+
if (retryResult.score === 1) {
|
|
1157
|
+
isCorrect = true;
|
|
1158
|
+
// Update response to the successful retry for logging
|
|
1159
|
+
response = retryResponse;
|
|
1160
|
+
// Update usage to the successful retry
|
|
1161
|
+
usage = retryResponse.totalUsage ?? retryResponse.usage;
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
1164
|
+
const didRetry = retryCount > 0;
|
|
1165
|
+
|
|
1166
|
+
// Run improved evaluation if improved question OR improved answer exists (unless --no-fixed)
|
|
1167
|
+
let improvedQuestion: string | undefined;
|
|
1168
|
+
let improvedAnswer: string | undefined;
|
|
1169
|
+
let improvedHypothesis: string | undefined;
|
|
1170
|
+
let improvedIsCorrect: boolean | undefined;
|
|
1171
|
+
let improvedUsage: typeof usage | undefined;
|
|
1172
|
+
|
|
1173
|
+
if (!options.skipFixed) {
|
|
1174
|
+
// Normalize: if only improvedAnswer exists, copy the original question to improvedQuestion
|
|
1175
|
+
// This simplifies the logic - we always run a "fixed" evaluation if either field is set
|
|
1176
|
+
improvedQuestion = meta.improvedQuestion ?? (meta.improvedAnswer ? meta.question : undefined);
|
|
1177
|
+
improvedAnswer = meta.improvedAnswer ?? meta.answer;
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1180
|
+
if (improvedQuestion) {
|
|
1181
|
+
// If the improved question is the same as the original (only answer changed),
|
|
1182
|
+
// reuse the vanilla response. Otherwise, run a new query.
|
|
1183
|
+
if (improvedQuestion === meta.question) {
|
|
1184
|
+
improvedHypothesis = response.text;
|
|
1185
|
+
} else {
|
|
1186
|
+
updateStatus(`Running improved question for ${meta.questionId}...`);
|
|
1187
|
+
|
|
1188
|
+
// Create a separate thread for the improved question evaluation
|
|
1189
|
+
const improvedThreadId = `eval_improved_${meta.questionId}_${Date.now()}`;
|
|
1190
|
+
|
|
1191
|
+
const improvedResponse = await withRateLimitRetry(
|
|
1192
|
+
() =>
|
|
1193
|
+
agent.generate(improvedQuestion, {
|
|
1194
|
+
threadId: improvedThreadId,
|
|
1195
|
+
resourceId: meta.resourceId,
|
|
1196
|
+
requestContext,
|
|
1197
|
+
modelSettings: {
|
|
1198
|
+
temperature: 0,
|
|
1199
|
+
},
|
|
1200
|
+
}),
|
|
1201
|
+
spinner,
|
|
1202
|
+
);
|
|
1203
|
+
|
|
1204
|
+
improvedHypothesis = improvedResponse.text;
|
|
1205
|
+
improvedUsage = improvedResponse.totalUsage ?? improvedResponse.usage;
|
|
1206
|
+
}
|
|
1207
|
+
|
|
1208
|
+
const improvedInput = JSON.stringify({
|
|
1209
|
+
question: improvedQuestion,
|
|
1210
|
+
answer: improvedAnswer,
|
|
1211
|
+
});
|
|
1212
|
+
|
|
1213
|
+
const improvedResult = await withRateLimitRetry(
|
|
1214
|
+
() => metric.measure(improvedInput, improvedHypothesis!),
|
|
1215
|
+
spinner,
|
|
1216
|
+
);
|
|
1217
|
+
improvedIsCorrect = improvedResult.score === 1;
|
|
1218
|
+
|
|
1219
|
+
// Retry improved version: always retry at least once, up to 5 times if requiresRetry is set
|
|
1220
|
+
const improvedMaxRetries = meta.requiresRetry ? maxRetries : 1;
|
|
1221
|
+
let improvedRetryCount = 0;
|
|
1222
|
+
while (!improvedIsCorrect && improvedRetryCount < improvedMaxRetries) {
|
|
1223
|
+
improvedRetryCount++;
|
|
1224
|
+
updateStatus(`Retry improved ${improvedRetryCount}/${improvedMaxRetries} for ${meta.questionId}...`);
|
|
1225
|
+
|
|
1226
|
+
const retryThreadId = `eval_improved_retry_${meta.questionId}_${improvedRetryCount}_${Date.now()}`;
|
|
1227
|
+
const retryResponse = await withRateLimitRetry(
|
|
1228
|
+
() =>
|
|
1229
|
+
agent.generate(improvedQuestion, {
|
|
1230
|
+
threadId: retryThreadId,
|
|
1231
|
+
resourceId: meta.resourceId,
|
|
1232
|
+
requestContext,
|
|
1233
|
+
modelSettings: {
|
|
1234
|
+
temperature: 0,
|
|
1235
|
+
},
|
|
1236
|
+
// Date is injected via DateInjector processor in official LongMemEval format
|
|
1237
|
+
}),
|
|
1238
|
+
spinner,
|
|
1239
|
+
);
|
|
1240
|
+
|
|
1241
|
+
const retryResult = await withRateLimitRetry(() => metric.measure(improvedInput, retryResponse.text), spinner);
|
|
1242
|
+
if (retryResult.score === 1) {
|
|
1243
|
+
improvedIsCorrect = true;
|
|
1244
|
+
improvedHypothesis = retryResponse.text;
|
|
1245
|
+
// Update improved usage to the successful retry
|
|
1246
|
+
improvedUsage = retryResponse.totalUsage ?? retryResponse.usage;
|
|
1247
|
+
}
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
375
1250
|
|
|
376
1251
|
const elapsed = ((Date.now() - questionStart) / 1000).toFixed(1);
|
|
377
1252
|
|
|
378
1253
|
const isOraSpinner = spinner && 'clear' in spinner;
|
|
379
1254
|
if (isOraSpinner) {
|
|
1255
|
+
// Show vanilla result (with retry indicator if applicable)
|
|
1256
|
+
const retryIndicator = didRetry
|
|
1257
|
+
? isCorrect
|
|
1258
|
+
? chalk.yellow(` (retry ${retryCount}/${maxRetries} ✓)`)
|
|
1259
|
+
: chalk.gray(` (retry ${retryCount}/${maxRetries} ✗)`)
|
|
1260
|
+
: '';
|
|
380
1261
|
console.log(
|
|
381
1262
|
chalk.blue(`▶ ${meta.questionId}`),
|
|
382
1263
|
chalk.gray(`(${meta.questionType})`),
|
|
383
1264
|
chalk[isCorrect ? 'green' : 'red'](`${isCorrect ? '✓' : '✗'}`),
|
|
1265
|
+
retryIndicator,
|
|
384
1266
|
chalk.gray(`${elapsed}s`),
|
|
385
1267
|
);
|
|
386
1268
|
if (!isCorrect) {
|
|
@@ -388,42 +1270,120 @@ Be specific rather than generic when the user has expressed clear preferences in
|
|
|
388
1270
|
console.log(chalk.gray(` A: "${response.text}"`));
|
|
389
1271
|
console.log(chalk.yellow(` Expected: "${meta.answer}"`));
|
|
390
1272
|
}
|
|
1273
|
+
|
|
1274
|
+
// Show improved result if applicable
|
|
1275
|
+
if (improvedQuestion) {
|
|
1276
|
+
// Show whether it's an improved question or just improved answer
|
|
1277
|
+
const label = meta.improvedQuestion ? 'improved Q' : 'improved A';
|
|
1278
|
+
console.log(
|
|
1279
|
+
chalk.cyan(` ↳ ${label}:`),
|
|
1280
|
+
chalk[improvedIsCorrect ? 'green' : 'red'](`${improvedIsCorrect ? '✓' : '✗'}`),
|
|
1281
|
+
);
|
|
1282
|
+
console.log(chalk.gray(` Q: "${improvedQuestion}"`));
|
|
1283
|
+
console.log(chalk.gray(` A: "${improvedHypothesis}"`));
|
|
1284
|
+
if (!improvedIsCorrect) {
|
|
1285
|
+
console.log(chalk.yellow(` Expected: "${improvedAnswer}"`));
|
|
1286
|
+
}
|
|
1287
|
+
}
|
|
391
1288
|
}
|
|
392
1289
|
|
|
1290
|
+
// Track when improved version performs worse than original
|
|
1291
|
+
const improvedRegression = isCorrect && improvedQuestion !== undefined && !improvedIsCorrect;
|
|
1292
|
+
|
|
393
1293
|
return {
|
|
394
1294
|
question_id: meta.questionId,
|
|
1295
|
+
question: meta.question,
|
|
1296
|
+
expected_answer: meta.answer,
|
|
395
1297
|
hypothesis: response.text,
|
|
396
1298
|
question_type: meta.questionType as QuestionType,
|
|
397
1299
|
is_correct: isCorrect,
|
|
1300
|
+
improved_question: improvedQuestion,
|
|
1301
|
+
improved_hypothesis: improvedHypothesis,
|
|
1302
|
+
improved_is_correct: improvedIsCorrect,
|
|
1303
|
+
has_improvement_info: !!(meta.improvedQuestion || meta.improvedAnswer || meta.improvementNote),
|
|
1304
|
+
improved_regression: improvedRegression,
|
|
1305
|
+
usage:
|
|
1306
|
+
usage && usage.inputTokens !== undefined
|
|
1307
|
+
? {
|
|
1308
|
+
inputTokens: usage.inputTokens,
|
|
1309
|
+
outputTokens: usage.outputTokens ?? 0,
|
|
1310
|
+
totalTokens: usage.totalTokens ?? usage.inputTokens + (usage.outputTokens ?? 0),
|
|
1311
|
+
}
|
|
1312
|
+
: undefined,
|
|
1313
|
+
improved_usage:
|
|
1314
|
+
improvedUsage && improvedUsage.inputTokens !== undefined
|
|
1315
|
+
? {
|
|
1316
|
+
inputTokens: improvedUsage.inputTokens,
|
|
1317
|
+
outputTokens: improvedUsage.outputTokens ?? 0,
|
|
1318
|
+
totalTokens: improvedUsage.totalTokens ?? improvedUsage.inputTokens + (improvedUsage.outputTokens ?? 0),
|
|
1319
|
+
}
|
|
1320
|
+
: undefined,
|
|
398
1321
|
};
|
|
399
1322
|
}
|
|
400
1323
|
|
|
401
|
-
private async
|
|
1324
|
+
private async saveFinalResults(
|
|
402
1325
|
runDir: string,
|
|
403
1326
|
results: EvaluationResult[],
|
|
404
1327
|
metrics: BenchmarkMetrics,
|
|
405
1328
|
options: RunOptions,
|
|
406
1329
|
): Promise<void> {
|
|
407
|
-
//
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
1330
|
+
// Note: results.jsonl is already saved incrementally during the run
|
|
1331
|
+
|
|
1332
|
+
// Save failures.json for re-preparation
|
|
1333
|
+
// Only include questions where BOTH original and improved failed (or no improved version exists)
|
|
1334
|
+
const failedQuestionIds = results
|
|
1335
|
+
.filter(r => {
|
|
1336
|
+
// If original passed, not a failure
|
|
1337
|
+
if (r.is_correct) return false;
|
|
1338
|
+
// If there's an improved version and it passed, not a failure
|
|
1339
|
+
if (r.improved_question !== undefined && r.improved_is_correct) return false;
|
|
1340
|
+
// Both failed (or no improved version)
|
|
1341
|
+
return true;
|
|
1342
|
+
})
|
|
1343
|
+
.map(r => r.question_id);
|
|
1344
|
+
if (failedQuestionIds.length > 0) {
|
|
1345
|
+
const failuresPath = join(runDir, 'failures.json');
|
|
1346
|
+
const failuresData = {
|
|
1347
|
+
questionIds: failedQuestionIds,
|
|
1348
|
+
runId: runDir.split('/').pop(),
|
|
1349
|
+
config: options.memoryConfig,
|
|
1350
|
+
dataset: options.dataset,
|
|
1351
|
+
timestamp: new Date().toISOString(),
|
|
1352
|
+
totalFailed: failedQuestionIds.length,
|
|
1353
|
+
totalQuestions: results.length,
|
|
1354
|
+
};
|
|
1355
|
+
await writeFile(failuresPath, JSON.stringify(failuresData, null, 2));
|
|
1356
|
+
console.log(chalk.gray(`Failures saved to: ${failuresPath}`));
|
|
1357
|
+
}
|
|
411
1358
|
|
|
412
1359
|
// Save metrics
|
|
413
1360
|
const metricsPath = join(runDir, 'metrics.json');
|
|
1361
|
+
const metricsConfigDef = getMemoryConfig(options.memoryConfig);
|
|
414
1362
|
const metricsData = {
|
|
415
1363
|
...metrics,
|
|
416
1364
|
config: {
|
|
417
1365
|
dataset: options.dataset,
|
|
418
|
-
model:
|
|
1366
|
+
model: metricsConfigDef.agentModel,
|
|
419
1367
|
memoryConfig: options.memoryConfig,
|
|
420
1368
|
subset: options.subset,
|
|
1369
|
+
// Store OM config for reproducibility
|
|
1370
|
+
...(metricsConfigDef.usesObservationalMemory && {
|
|
1371
|
+
observationalMemoryConfig: {
|
|
1372
|
+
scope: 'resource',
|
|
1373
|
+
messageTokens: OBSERVATIONAL_MEMORY_DEFAULTS.observation.messageTokens,
|
|
1374
|
+
observationTokens: OBSERVATIONAL_MEMORY_DEFAULTS.reflection.observationTokens,
|
|
1375
|
+
recognizePatterns: false,
|
|
1376
|
+
},
|
|
1377
|
+
}),
|
|
421
1378
|
},
|
|
422
1379
|
timestamp: new Date().toISOString(),
|
|
423
1380
|
};
|
|
424
1381
|
await writeFile(metricsPath, JSON.stringify(metricsData, null, 2));
|
|
425
1382
|
|
|
426
|
-
console.log(
|
|
1383
|
+
console.log(
|
|
1384
|
+
chalk.gray(`
|
|
1385
|
+
Results saved to: ${runDir}`),
|
|
1386
|
+
);
|
|
427
1387
|
}
|
|
428
1388
|
|
|
429
1389
|
private calculateMetrics(results: EvaluationResult[]): BenchmarkMetrics {
|
|
@@ -435,15 +1395,33 @@ Be specific rather than generic when the user has expressed clear preferences in
|
|
|
435
1395
|
correct_answers: 0,
|
|
436
1396
|
abstention_correct: 0,
|
|
437
1397
|
abstention_total: 0,
|
|
1398
|
+
// "Fixed" metrics - uses improved_is_correct where available, otherwise is_correct
|
|
1399
|
+
improved_accuracy: undefined,
|
|
1400
|
+
improved_correct: 0,
|
|
1401
|
+
improved_total: 0,
|
|
1402
|
+
fixed_accuracy_by_type: {} as Record<QuestionType, { correct: number; total: number; accuracy: number }>,
|
|
1403
|
+
fixed_overall_accuracy: undefined,
|
|
438
1404
|
};
|
|
439
1405
|
|
|
1406
|
+
// Check if any results have improved questions
|
|
1407
|
+
const hasAnyImprovedQuestions = results.some(r => r.improved_question !== undefined);
|
|
1408
|
+
|
|
440
1409
|
// Calculate overall metrics
|
|
441
1410
|
for (const result of results) {
|
|
1411
|
+
// Vanilla metrics (original question only)
|
|
442
1412
|
if (result.is_correct) {
|
|
443
1413
|
metrics.correct_answers++;
|
|
444
1414
|
}
|
|
445
1415
|
|
|
446
|
-
// Track
|
|
1416
|
+
// Track how many questions have improved versions
|
|
1417
|
+
if (result.improved_question !== undefined) {
|
|
1418
|
+
metrics.improved_total = (metrics.improved_total || 0) + 1;
|
|
1419
|
+
if (result.improved_is_correct) {
|
|
1420
|
+
metrics.improved_correct = (metrics.improved_correct || 0) + 1;
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
// Track by question type (vanilla)
|
|
447
1425
|
if (result.question_type) {
|
|
448
1426
|
const type = result.question_type;
|
|
449
1427
|
if (!metrics.accuracy_by_type[type]) {
|
|
@@ -453,6 +1431,21 @@ Be specific rather than generic when the user has expressed clear preferences in
|
|
|
453
1431
|
if (result.is_correct) {
|
|
454
1432
|
metrics.accuracy_by_type[type].correct++;
|
|
455
1433
|
}
|
|
1434
|
+
|
|
1435
|
+
// Track "fixed" metrics by type (use improved result if available, otherwise vanilla)
|
|
1436
|
+
if (hasAnyImprovedQuestions) {
|
|
1437
|
+
if (!metrics.fixed_accuracy_by_type![type]) {
|
|
1438
|
+
metrics.fixed_accuracy_by_type![type] = { correct: 0, total: 0, accuracy: 0 };
|
|
1439
|
+
}
|
|
1440
|
+
metrics.fixed_accuracy_by_type![type].total++;
|
|
1441
|
+
|
|
1442
|
+
// Fixed score: correct if EITHER original OR improved passes
|
|
1443
|
+
// This ensures we don't penalize when improved question regresses
|
|
1444
|
+
const isCorrectFixed = result.is_correct || (result.improved_is_correct ?? false);
|
|
1445
|
+
if (isCorrectFixed) {
|
|
1446
|
+
metrics.fixed_accuracy_by_type![type].correct++;
|
|
1447
|
+
}
|
|
1448
|
+
}
|
|
456
1449
|
}
|
|
457
1450
|
|
|
458
1451
|
// Track abstention separately
|
|
@@ -462,9 +1455,27 @@ Be specific rather than generic when the user has expressed clear preferences in
|
|
|
462
1455
|
metrics.abstention_correct = (metrics.abstention_correct || 0) + 1;
|
|
463
1456
|
}
|
|
464
1457
|
}
|
|
1458
|
+
|
|
1459
|
+
// Aggregate token usage
|
|
1460
|
+
if (result.usage) {
|
|
1461
|
+
if (!metrics.total_usage) {
|
|
1462
|
+
metrics.total_usage = { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
|
|
1463
|
+
}
|
|
1464
|
+
metrics.total_usage.inputTokens += result.usage.inputTokens;
|
|
1465
|
+
metrics.total_usage.outputTokens += result.usage.outputTokens;
|
|
1466
|
+
metrics.total_usage.totalTokens += result.usage.totalTokens;
|
|
1467
|
+
}
|
|
1468
|
+
if (result.improved_usage) {
|
|
1469
|
+
if (!metrics.improved_total_usage) {
|
|
1470
|
+
metrics.improved_total_usage = { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
|
|
1471
|
+
}
|
|
1472
|
+
metrics.improved_total_usage.inputTokens += result.improved_usage.inputTokens;
|
|
1473
|
+
metrics.improved_total_usage.outputTokens += result.improved_usage.outputTokens;
|
|
1474
|
+
metrics.improved_total_usage.totalTokens += result.improved_usage.totalTokens;
|
|
1475
|
+
}
|
|
465
1476
|
}
|
|
466
1477
|
|
|
467
|
-
// Calculate per-type accuracies
|
|
1478
|
+
// Calculate per-type accuracies (vanilla)
|
|
468
1479
|
for (const type in metrics.accuracy_by_type) {
|
|
469
1480
|
const typeMetrics = metrics.accuracy_by_type[type as QuestionType];
|
|
470
1481
|
if (typeMetrics) {
|
|
@@ -472,40 +1483,54 @@ Be specific rather than generic when the user has expressed clear preferences in
|
|
|
472
1483
|
}
|
|
473
1484
|
}
|
|
474
1485
|
|
|
1486
|
+
// Calculate per-type accuracies (fixed)
|
|
1487
|
+
if (hasAnyImprovedQuestions && metrics.fixed_accuracy_by_type) {
|
|
1488
|
+
for (const type in metrics.fixed_accuracy_by_type) {
|
|
1489
|
+
const typeMetrics = metrics.fixed_accuracy_by_type[type as QuestionType];
|
|
1490
|
+
if (typeMetrics) {
|
|
1491
|
+
typeMetrics.accuracy = typeMetrics.total > 0 ? typeMetrics.correct / typeMetrics.total : 0;
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1494
|
+
}
|
|
1495
|
+
|
|
475
1496
|
if (metrics.abstention_total && metrics.abstention_total > 0) {
|
|
476
1497
|
metrics.abstention_accuracy = (metrics.abstention_correct || 0) / metrics.abstention_total;
|
|
477
1498
|
}
|
|
478
1499
|
|
|
479
|
-
// Calculate overall accuracy as average of all question type accuracies (
|
|
1500
|
+
// Calculate overall accuracy as average of all question type accuracies (vanilla)
|
|
480
1501
|
const allTypeAccuracies = Object.values(metrics.accuracy_by_type).map(t => t.accuracy);
|
|
481
|
-
|
|
482
|
-
// Debug: Log the exact values being averaged
|
|
483
|
-
console.log('\nDebug - Question type accuracies being averaged:');
|
|
484
|
-
Object.entries(metrics.accuracy_by_type).forEach(([type, data]) => {
|
|
485
|
-
console.log(` ${type}: ${data.accuracy} (${data.correct}/${data.total})`);
|
|
486
|
-
});
|
|
487
|
-
console.log(` Sum: ${allTypeAccuracies.reduce((sum, acc) => sum + acc, 0)}`);
|
|
488
|
-
console.log(` Count: ${allTypeAccuracies.length}`);
|
|
489
|
-
|
|
490
1502
|
metrics.overall_accuracy =
|
|
491
1503
|
allTypeAccuracies.length > 0
|
|
492
1504
|
? allTypeAccuracies.reduce((sum, acc) => sum + acc, 0) / allTypeAccuracies.length
|
|
493
1505
|
: 0;
|
|
494
1506
|
|
|
495
|
-
|
|
496
|
-
|
|
1507
|
+
// Calculate fixed overall accuracy
|
|
1508
|
+
if (hasAnyImprovedQuestions && metrics.fixed_accuracy_by_type) {
|
|
1509
|
+
const fixedTypeAccuracies = Object.values(metrics.fixed_accuracy_by_type).map(t => t.accuracy);
|
|
1510
|
+
metrics.fixed_overall_accuracy =
|
|
1511
|
+
fixedTypeAccuracies.length > 0
|
|
1512
|
+
? fixedTypeAccuracies.reduce((sum, acc) => sum + acc, 0) / fixedTypeAccuracies.length
|
|
1513
|
+
: 0;
|
|
1514
|
+
}
|
|
497
1515
|
|
|
498
1516
|
return metrics;
|
|
499
1517
|
}
|
|
500
1518
|
|
|
501
|
-
private displayMetrics(metrics: BenchmarkMetrics, options?: RunOptions): void {
|
|
502
|
-
console.log(
|
|
1519
|
+
private displayMetrics(metrics: BenchmarkMetrics, options?: RunOptions, configDef?: MemoryConfigDefinition): void {
|
|
1520
|
+
console.log(
|
|
1521
|
+
chalk.bold(`
|
|
1522
|
+
📊 Benchmark Results
|
|
1523
|
+
`),
|
|
1524
|
+
);
|
|
503
1525
|
|
|
504
1526
|
// Display configuration if provided
|
|
505
1527
|
if (options) {
|
|
506
|
-
console.log(
|
|
1528
|
+
console.log(
|
|
1529
|
+
chalk.bold(`Configuration:
|
|
1530
|
+
`),
|
|
1531
|
+
);
|
|
507
1532
|
console.log(chalk.gray('Dataset:'), chalk.cyan(options.dataset));
|
|
508
|
-
console.log(chalk.gray('Model:'), chalk.cyan(
|
|
1533
|
+
console.log(chalk.gray('Model:'), chalk.cyan(configDef?.agentModel ?? 'gpt-4o'));
|
|
509
1534
|
console.log(chalk.gray('Memory Config:'), chalk.cyan(options.memoryConfig));
|
|
510
1535
|
if (options.subset) {
|
|
511
1536
|
console.log(chalk.gray('Subset:'), chalk.cyan(`${options.subset} questions`));
|
|
@@ -517,37 +1542,131 @@ Be specific rather than generic when the user has expressed clear preferences in
|
|
|
517
1542
|
console.log();
|
|
518
1543
|
}
|
|
519
1544
|
|
|
1545
|
+
// Check if we have fixed metrics to display
|
|
1546
|
+
const hasFixedMetrics = metrics.fixed_accuracy_by_type && Object.keys(metrics.fixed_accuracy_by_type).length > 0;
|
|
1547
|
+
|
|
520
1548
|
// Question type breakdown
|
|
521
1549
|
console.log(chalk.bold('Accuracy by Question Type:'));
|
|
522
1550
|
|
|
523
1551
|
// Sort question types alphabetically
|
|
524
1552
|
const sortedTypes = Object.entries(metrics.accuracy_by_type).sort(([a], [b]) => a.localeCompare(b));
|
|
525
1553
|
|
|
526
|
-
//
|
|
1554
|
+
// Helper to create progress bar
|
|
1555
|
+
const createBar = (accuracy: number, length: number = 20) => {
|
|
1556
|
+
const filledLength = Math.round(accuracy * length);
|
|
1557
|
+
return '█'.repeat(filledLength) + '░'.repeat(length - filledLength);
|
|
1558
|
+
};
|
|
1559
|
+
|
|
1560
|
+
// Helper to get color based on accuracy
|
|
1561
|
+
const getColor = (accuracy: number) => (accuracy >= 0.8 ? 'green' : accuracy >= 0.6 ? 'yellow' : 'red');
|
|
1562
|
+
|
|
1563
|
+
// Display regular question types (vanilla)
|
|
527
1564
|
for (const [type, typeMetrics] of sortedTypes) {
|
|
528
1565
|
const { correct, total, accuracy } = typeMetrics;
|
|
529
|
-
const typeColor = accuracy
|
|
530
|
-
|
|
531
|
-
// Create a simple progress bar
|
|
532
|
-
const barLength = 20;
|
|
533
|
-
const filledLength = Math.round(accuracy * barLength);
|
|
534
|
-
const bar = '█'.repeat(filledLength) + '░'.repeat(barLength - filledLength);
|
|
1566
|
+
const typeColor = getColor(accuracy);
|
|
535
1567
|
|
|
536
1568
|
console.log(
|
|
537
1569
|
chalk.gray(` ${type.padEnd(25)}:`),
|
|
538
1570
|
chalk[typeColor](`${(accuracy * 100).toFixed(1).padStart(5)}%`),
|
|
539
|
-
chalk.gray(`[${
|
|
1571
|
+
chalk.gray(`[${createBar(accuracy)}]`),
|
|
540
1572
|
chalk.gray(`(${correct}/${total})`),
|
|
541
1573
|
);
|
|
1574
|
+
|
|
1575
|
+
// If we have fixed metrics, show the fixed version right after
|
|
1576
|
+
if (hasFixedMetrics && metrics.fixed_accuracy_by_type![type as QuestionType]) {
|
|
1577
|
+
const fixedTypeMetrics = metrics.fixed_accuracy_by_type![type as QuestionType];
|
|
1578
|
+
const fixedColor = getColor(fixedTypeMetrics.accuracy);
|
|
1579
|
+
console.log(
|
|
1580
|
+
chalk.gray(` ${(type + ' (fixed)').padEnd(25)}:`),
|
|
1581
|
+
chalk[fixedColor](`${(fixedTypeMetrics.accuracy * 100).toFixed(1).padStart(5)}%`),
|
|
1582
|
+
chalk.gray(`[${createBar(fixedTypeMetrics.accuracy)}]`),
|
|
1583
|
+
chalk.gray(`(${fixedTypeMetrics.correct}/${fixedTypeMetrics.total})`),
|
|
1584
|
+
);
|
|
1585
|
+
}
|
|
542
1586
|
}
|
|
543
1587
|
|
|
544
1588
|
console.log();
|
|
545
|
-
|
|
546
|
-
|
|
1589
|
+
|
|
1590
|
+
// Overall accuracy (vanilla)
|
|
1591
|
+
const accuracyColor = getColor(metrics.overall_accuracy);
|
|
547
1592
|
console.log(
|
|
548
|
-
chalk.bold('Overall Accuracy:'),
|
|
1593
|
+
chalk.bold('Overall Accuracy: '),
|
|
549
1594
|
chalk[accuracyColor](`${(metrics.overall_accuracy * 100).toFixed(2)}%`),
|
|
550
1595
|
chalk.gray(`(average of ${Object.keys(metrics.accuracy_by_type).length} question types)`),
|
|
551
1596
|
);
|
|
1597
|
+
|
|
1598
|
+
// Overall accuracy (fixed) - shown if any improved questions exist
|
|
1599
|
+
if (hasFixedMetrics && metrics.fixed_overall_accuracy !== undefined) {
|
|
1600
|
+
const fixedAccuracyColor = getColor(metrics.fixed_overall_accuracy);
|
|
1601
|
+
console.log(
|
|
1602
|
+
chalk.bold('Overall Accuracy (fixed):'),
|
|
1603
|
+
chalk[fixedAccuracyColor](`${(metrics.fixed_overall_accuracy * 100).toFixed(2)}%`),
|
|
1604
|
+
chalk.gray(`(${metrics.improved_total} questions clarified)`),
|
|
1605
|
+
);
|
|
1606
|
+
}
|
|
1607
|
+
|
|
1608
|
+
// Token usage summary
|
|
1609
|
+
if (metrics.total_usage) {
|
|
1610
|
+
console.log();
|
|
1611
|
+
console.log(chalk.bold('Token Usage:'));
|
|
1612
|
+
const formatTokens = (n: number) => n.toLocaleString();
|
|
1613
|
+
console.log(
|
|
1614
|
+
chalk.gray(' Input tokens: '),
|
|
1615
|
+
chalk.cyan(formatTokens(metrics.total_usage.inputTokens)),
|
|
1616
|
+
chalk.gray(
|
|
1617
|
+
`(avg ${formatTokens(Math.round(metrics.total_usage.inputTokens / metrics.total_questions))}/question)`,
|
|
1618
|
+
),
|
|
1619
|
+
);
|
|
1620
|
+
console.log(chalk.gray(' Output tokens:'), chalk.cyan(formatTokens(metrics.total_usage.outputTokens)));
|
|
1621
|
+
console.log(chalk.gray(' Total tokens: '), chalk.cyan(formatTokens(metrics.total_usage.totalTokens)));
|
|
1622
|
+
|
|
1623
|
+
if (metrics.improved_total_usage) {
|
|
1624
|
+
console.log(
|
|
1625
|
+
chalk.gray(' Improved input tokens: '),
|
|
1626
|
+
chalk.cyan(formatTokens(metrics.improved_total_usage.inputTokens)),
|
|
1627
|
+
);
|
|
1628
|
+
}
|
|
1629
|
+
}
|
|
1630
|
+
}
|
|
1631
|
+
|
|
1632
|
+
private displayUninvestigatedFailures(results: EvaluationResult[]): void {
|
|
1633
|
+
// Find failures that have no improvement info (not yet investigated)
|
|
1634
|
+
const uninvestigatedFailures = results.filter(r => r.is_correct === false && !r.has_improvement_info);
|
|
1635
|
+
|
|
1636
|
+
if (uninvestigatedFailures.length === 0) {
|
|
1637
|
+
return;
|
|
1638
|
+
}
|
|
1639
|
+
|
|
1640
|
+
console.log(
|
|
1641
|
+
chalk.yellow(`
|
|
1642
|
+
🔍 Failures for Investigation (${uninvestigatedFailures.length})
|
|
1643
|
+
`),
|
|
1644
|
+
);
|
|
1645
|
+
console.log(
|
|
1646
|
+
chalk.gray('These questions failed and have no improved_question, improved_answer, or improvement_note:\n'),
|
|
1647
|
+
);
|
|
1648
|
+
|
|
1649
|
+
// Group by question type for easier review
|
|
1650
|
+
const byType = new Map<string, EvaluationResult[]>();
|
|
1651
|
+
for (const result of uninvestigatedFailures) {
|
|
1652
|
+
const type = result.question_type || 'unknown';
|
|
1653
|
+
if (!byType.has(type)) {
|
|
1654
|
+
byType.set(type, []);
|
|
1655
|
+
}
|
|
1656
|
+
byType.get(type)!.push(result);
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1659
|
+
// Display grouped by type
|
|
1660
|
+
for (const [type, failures] of Array.from(byType.entries()).sort(([a], [b]) => a.localeCompare(b))) {
|
|
1661
|
+
console.log(chalk.cyan(` ${type}:`));
|
|
1662
|
+
for (const result of failures) {
|
|
1663
|
+
console.log(chalk.gray(` - ${result.question_id}`));
|
|
1664
|
+
console.log(chalk.gray(` Q: "${result.question}"`));
|
|
1665
|
+
console.log(chalk.gray(` A: "${result.hypothesis}"`));
|
|
1666
|
+
console.log(chalk.yellow(` Expected: "${result.expected_answer}"`));
|
|
1667
|
+
}
|
|
1668
|
+
}
|
|
1669
|
+
|
|
1670
|
+
console.log();
|
|
552
1671
|
}
|
|
553
1672
|
}
|