@mastra/longmemeval 0.0.0-error-handler-fix-20251020202607 ā 0.0.0-execa-dynamic-import-20260304221256
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +847 -6
- package/LICENSE.md +15 -0
- package/docs/INVESTIGATION_WORKFLOW.md +314 -0
- package/package.json +34 -26
- package/scripts/download.ts +22 -1
- package/scripts/generate-wm-templates.ts +2 -1
- package/src/cli.ts +967 -155
- package/src/commands/__tests__/prepare.test.ts +11 -6
- package/src/commands/__tests__/run.test.ts +10 -8
- package/src/commands/clean.ts +147 -0
- package/src/commands/deterministic-ids.ts +126 -0
- package/src/commands/find-prohibited.ts +299 -0
- package/src/commands/investigate.ts +2437 -0
- package/src/commands/list-partial.ts +137 -0
- package/src/commands/obscure-thread-ids.ts +237 -0
- package/src/commands/partial-results.ts +198 -0
- package/src/commands/precompute-embeddings.ts +302 -0
- package/src/commands/prepare.ts +732 -105
- package/src/commands/reconcile.ts +383 -0
- package/src/commands/run.ts +1240 -121
- package/src/commands/sessions.ts +126 -0
- package/src/commands/sync.ts +152 -0
- package/src/commands/test-cleaned.ts +53 -0
- package/src/commands/test-incremental.ts +120 -0
- package/src/commands/test-messages.ts +95 -0
- package/src/commands/test-prohibited.ts +244 -0
- package/src/commands/test-thread.ts +136 -0
- package/src/commands/tokens.ts +625 -0
- package/src/config.ts +1253 -72
- package/src/data/types.ts +48 -6
- package/src/embeddings/cached-openai-embedding-model.ts +0 -1
- package/src/evaluation/longmemeval-metric.ts +186 -103
- package/src/processors/content-sanitizer.ts +153 -0
- package/src/processors/date-injector.ts +111 -0
- package/src/processors/index.ts +3 -0
- package/src/processors/observation-semantic-filter.ts +1142 -0
- package/src/retry-model.ts +5 -3
- package/src/storage/__tests__/benchmark-store.test.ts +18 -35
- package/src/storage/benchmark-store.ts +140 -153
- package/src/storage/benchmark-vector.ts +1 -1
- package/src/storage/index.ts +1 -0
- package/src/storage/persistable-inmemory.ts +133 -0
- package/src/test-utils/mock-model.ts +35 -0
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { readdir, readFile } from 'fs/promises';
|
|
2
|
+
import { join } from 'path';
|
|
3
|
+
import { existsSync } from 'fs';
|
|
4
|
+
import chalk from 'chalk';
|
|
5
|
+
|
|
6
|
+
interface ListPartialOptions {
|
|
7
|
+
preparedDataDir?: string;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
interface PartialQuestion {
|
|
11
|
+
questionId: string;
|
|
12
|
+
processedSessions: number;
|
|
13
|
+
totalSessions?: number;
|
|
14
|
+
failed: boolean;
|
|
15
|
+
failedAt?: string;
|
|
16
|
+
error?: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Lists partially prepared questions (have progress.json but no meta.json or failed)
|
|
21
|
+
*/
|
|
22
|
+
export class ListPartialCommand {
|
|
23
|
+
private preparedDataDir: string;
|
|
24
|
+
|
|
25
|
+
constructor(options: ListPartialOptions) {
|
|
26
|
+
this.preparedDataDir =
|
|
27
|
+
options.preparedDataDir || join(process.cwd(), 'prepared-data', 'longmemeval_s', 'observational-memory');
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
async run(): Promise<void> {
|
|
31
|
+
console.log(chalk.blue('\nš Scanning for partially prepared questions...\n'));
|
|
32
|
+
|
|
33
|
+
if (!existsSync(this.preparedDataDir)) {
|
|
34
|
+
console.log(chalk.yellow(`Directory not found: ${this.preparedDataDir}`));
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const questionDirs = (await readdir(this.preparedDataDir)).filter(d => !d.startsWith('.'));
|
|
39
|
+
|
|
40
|
+
const partialQuestions: PartialQuestion[] = [];
|
|
41
|
+
const failedQuestions: PartialQuestion[] = [];
|
|
42
|
+
|
|
43
|
+
for (const questionId of questionDirs) {
|
|
44
|
+
const questionDir = join(this.preparedDataDir, questionId);
|
|
45
|
+
const progressPath = join(questionDir, 'progress.json');
|
|
46
|
+
const metaPath = join(questionDir, 'meta.json');
|
|
47
|
+
|
|
48
|
+
// Skip if fully prepared (has meta.json)
|
|
49
|
+
if (existsSync(metaPath)) {
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Check if has progress.json
|
|
54
|
+
if (existsSync(progressPath)) {
|
|
55
|
+
try {
|
|
56
|
+
const progress = JSON.parse(await readFile(progressPath, 'utf-8'));
|
|
57
|
+
const info: PartialQuestion = {
|
|
58
|
+
questionId,
|
|
59
|
+
processedSessions: progress.processedSessionIds?.length || 0,
|
|
60
|
+
totalSessions: progress.totalSessions,
|
|
61
|
+
failed: progress.failed || false,
|
|
62
|
+
failedAt: progress.failedAt,
|
|
63
|
+
error: progress.error,
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
if (info.failed) {
|
|
67
|
+
failedQuestions.push(info);
|
|
68
|
+
} else {
|
|
69
|
+
partialQuestions.push(info);
|
|
70
|
+
}
|
|
71
|
+
} catch (e) {
|
|
72
|
+
// Corrupted progress file
|
|
73
|
+
partialQuestions.push({
|
|
74
|
+
questionId,
|
|
75
|
+
processedSessions: 0,
|
|
76
|
+
failed: false,
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Display results
|
|
83
|
+
if (failedQuestions.length > 0) {
|
|
84
|
+
console.log(chalk.red(`ā Failed questions (${failedQuestions.length}):\n`));
|
|
85
|
+
for (const q of failedQuestions) {
|
|
86
|
+
const progress = q.totalSessions
|
|
87
|
+
? `${q.processedSessions}/${q.totalSessions} sessions`
|
|
88
|
+
: `${q.processedSessions} sessions`;
|
|
89
|
+
console.log(` ${chalk.red(q.questionId)} - ${progress}`);
|
|
90
|
+
if (q.error) {
|
|
91
|
+
console.log(chalk.gray(` Error: ${q.error}`));
|
|
92
|
+
}
|
|
93
|
+
if (q.failedAt) {
|
|
94
|
+
console.log(chalk.gray(` Failed at: ${q.failedAt}`));
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
console.log();
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
if (partialQuestions.length > 0) {
|
|
101
|
+
console.log(chalk.yellow(`ā³ Partially prepared questions (${partialQuestions.length}):\n`));
|
|
102
|
+
for (const q of partialQuestions) {
|
|
103
|
+
const progress = q.totalSessions
|
|
104
|
+
? `${q.processedSessions}/${q.totalSessions} sessions`
|
|
105
|
+
: `${q.processedSessions} sessions`;
|
|
106
|
+
console.log(` ${chalk.yellow(q.questionId)} - ${progress}`);
|
|
107
|
+
}
|
|
108
|
+
console.log();
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
if (failedQuestions.length === 0 && partialQuestions.length === 0) {
|
|
112
|
+
console.log(chalk.green('ā No partially prepared or failed questions found.\n'));
|
|
113
|
+
} else {
|
|
114
|
+
// Print summary with actionable commands
|
|
115
|
+
console.log(chalk.blue('ā'.repeat(50)));
|
|
116
|
+
console.log(chalk.white('\nTo resume preparation:'));
|
|
117
|
+
|
|
118
|
+
const allIds = [...failedQuestions, ...partialQuestions].map(q => q.questionId);
|
|
119
|
+
if (allIds.length === 1) {
|
|
120
|
+
console.log(chalk.gray(` pnpm prepare -q ${allIds[0]}`));
|
|
121
|
+
} else {
|
|
122
|
+
console.log(chalk.gray(` pnpm prepare # will auto-resume all`));
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
console.log(chalk.white('\nTo clean and re-prepare:'));
|
|
126
|
+
if (allIds.length <= 3) {
|
|
127
|
+
for (const id of allIds) {
|
|
128
|
+
console.log(chalk.gray(` pnpm clean -q ${id} && pnpm prepare -q ${id}`));
|
|
129
|
+
}
|
|
130
|
+
} else {
|
|
131
|
+
console.log(chalk.gray(` # Clean specific questions:`));
|
|
132
|
+
console.log(chalk.gray(` pnpm clean -q <question_id>`));
|
|
133
|
+
}
|
|
134
|
+
console.log();
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import { join } from 'path';
|
|
3
|
+
import { readdir, readFile, writeFile } from 'fs/promises';
|
|
4
|
+
import { existsSync } from 'fs';
|
|
5
|
+
import xxhash, { type XXHashAPI } from 'xxhash-wasm';
|
|
6
|
+
|
|
7
|
+
import { DatasetLoader } from '../data/loader';
|
|
8
|
+
import type { DatasetType } from '../data/types';
|
|
9
|
+
|
|
10
|
+
// Lazy-loaded hasher instance (same pattern as ObservationalMemory)
|
|
11
|
+
let hasherPromise: Promise<XXHashAPI> | null = null;
|
|
12
|
+
|
|
13
|
+
async function getHasher(): Promise<XXHashAPI> {
|
|
14
|
+
if (!hasherPromise) {
|
|
15
|
+
hasherPromise = xxhash();
|
|
16
|
+
}
|
|
17
|
+
return hasherPromise;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface ObscureThreadIdsOptions {
|
|
21
|
+
dataset: DatasetType;
|
|
22
|
+
memoryConfig: string;
|
|
23
|
+
preparedDataDir: string;
|
|
24
|
+
dryRun?: boolean;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Hash a thread ID using xxhash to create an opaque, non-reversible identifier.
|
|
29
|
+
* This prevents LLMs from recognizing patterns like "answer_" in thread IDs.
|
|
30
|
+
* Uses the same xxhash-wasm h32ToString as the runtime ObservationalMemory.
|
|
31
|
+
*/
|
|
32
|
+
async function hashThreadId(threadId: string): Promise<string> {
|
|
33
|
+
const hasher = await getHasher();
|
|
34
|
+
return hasher.h32ToString(threadId);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Replace thread IDs in a string with their hashed versions.
|
|
39
|
+
* Handles XML patterns:
|
|
40
|
+
* - <thread id="...">
|
|
41
|
+
* - <other-conversation id="...">
|
|
42
|
+
*/
|
|
43
|
+
function replaceThreadIdsInString(
|
|
44
|
+
content: string,
|
|
45
|
+
threadIdMap: Map<string, string>,
|
|
46
|
+
): { content: string; replacements: number } {
|
|
47
|
+
let replacements = 0;
|
|
48
|
+
|
|
49
|
+
// Replace <thread id="..."> patterns
|
|
50
|
+
let result = content.replace(/<thread\s+id="([^"]+)"/g, (match, threadId) => {
|
|
51
|
+
const hashed = threadIdMap.get(threadId);
|
|
52
|
+
if (hashed && hashed !== threadId) {
|
|
53
|
+
replacements++;
|
|
54
|
+
return `<thread id="${hashed}"`;
|
|
55
|
+
}
|
|
56
|
+
return match;
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
// Replace <other-conversation id="..."> patterns
|
|
60
|
+
result = result.replace(/<other-conversation\s+id="([^"]+)"/g, (match, threadId) => {
|
|
61
|
+
const hashed = threadIdMap.get(threadId);
|
|
62
|
+
if (hashed && hashed !== threadId) {
|
|
63
|
+
replacements++;
|
|
64
|
+
return `<other-conversation id="${hashed}"`;
|
|
65
|
+
}
|
|
66
|
+
return match;
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
return { content: result, replacements };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Process an om.json file: parse JSON, replace thread IDs in observation fields,
|
|
74
|
+
* and return the modified JSON.
|
|
75
|
+
*
|
|
76
|
+
* Structure:
|
|
77
|
+
* observationalMemory[i] = [resourceKey, innerArray]
|
|
78
|
+
* innerArray = [recordKey, record]
|
|
79
|
+
* record.activeObservations contains the observation text with <thread id="..."> tags
|
|
80
|
+
*/
|
|
81
|
+
function processOmJson(
|
|
82
|
+
jsonContent: string,
|
|
83
|
+
threadIdMap: Map<string, string>,
|
|
84
|
+
): { content: string; replacements: number } {
|
|
85
|
+
let totalReplacements = 0;
|
|
86
|
+
|
|
87
|
+
try {
|
|
88
|
+
const data = JSON.parse(jsonContent);
|
|
89
|
+
|
|
90
|
+
// observationalMemory is an array of [resourceKey, [recordKey, record]] pairs
|
|
91
|
+
if (data.observationalMemory && Array.isArray(data.observationalMemory)) {
|
|
92
|
+
for (const outerEntry of data.observationalMemory) {
|
|
93
|
+
if (Array.isArray(outerEntry) && outerEntry.length === 2) {
|
|
94
|
+
const innerArray = outerEntry[1]; // [recordKey, record]
|
|
95
|
+
|
|
96
|
+
if (Array.isArray(innerArray) && innerArray.length === 2) {
|
|
97
|
+
const record = innerArray[1]; // The actual OM record
|
|
98
|
+
|
|
99
|
+
// Replace in activeObservations
|
|
100
|
+
if (record.activeObservations && typeof record.activeObservations === 'string') {
|
|
101
|
+
const { content, replacements } = replaceThreadIdsInString(record.activeObservations, threadIdMap);
|
|
102
|
+
record.activeObservations = content;
|
|
103
|
+
totalReplacements += replacements;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// Replace in bufferedObservations (if present)
|
|
107
|
+
if (record.bufferedObservations && typeof record.bufferedObservations === 'string') {
|
|
108
|
+
const { content, replacements } = replaceThreadIdsInString(record.bufferedObservations, threadIdMap);
|
|
109
|
+
record.bufferedObservations = content;
|
|
110
|
+
totalReplacements += replacements;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
return {
|
|
118
|
+
content: JSON.stringify(data, null, 2),
|
|
119
|
+
replacements: totalReplacements,
|
|
120
|
+
};
|
|
121
|
+
} catch (error) {
|
|
122
|
+
// If JSON parsing fails, return original content
|
|
123
|
+
console.error(`Failed to parse JSON: ${error}`);
|
|
124
|
+
return { content: jsonContent, replacements: 0 };
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export class ObscureThreadIdsCommand {
|
|
129
|
+
async run(options: ObscureThreadIdsOptions): Promise<void> {
|
|
130
|
+
const { dataset, memoryConfig, preparedDataDir, dryRun } = options;
|
|
131
|
+
|
|
132
|
+
console.log(chalk.bold('\nš Obscure Thread IDs in Prepared Data\n'));
|
|
133
|
+
console.log(chalk.gray(`Dataset: ${dataset}`));
|
|
134
|
+
console.log(chalk.gray(`Memory config: ${memoryConfig}`));
|
|
135
|
+
console.log(chalk.gray(`Prepared data: ${preparedDataDir}`));
|
|
136
|
+
if (dryRun) {
|
|
137
|
+
console.log(chalk.yellow('\nā ļø DRY RUN - no files will be modified\n'));
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Load the source dataset to get all session IDs
|
|
141
|
+
console.log(chalk.gray(`\nLoading dataset to extract session IDs...`));
|
|
142
|
+
const loader = new DatasetLoader();
|
|
143
|
+
const questions = await loader.loadDataset(dataset);
|
|
144
|
+
|
|
145
|
+
// Build a map of all session IDs -> hashed versions
|
|
146
|
+
const threadIdMap = new Map<string, string>();
|
|
147
|
+
let sessionCount = 0;
|
|
148
|
+
|
|
149
|
+
for (const question of questions) {
|
|
150
|
+
// Each question has haystack_session_ids
|
|
151
|
+
if (question.haystack_session_ids) {
|
|
152
|
+
for (const sessionId of question.haystack_session_ids) {
|
|
153
|
+
if (sessionId && !threadIdMap.has(sessionId)) {
|
|
154
|
+
threadIdMap.set(sessionId, await hashThreadId(sessionId));
|
|
155
|
+
sessionCount++;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
// Also include answer_session_ids
|
|
160
|
+
if (question.answer_session_ids) {
|
|
161
|
+
for (const sessionId of question.answer_session_ids) {
|
|
162
|
+
if (sessionId && !threadIdMap.has(sessionId)) {
|
|
163
|
+
threadIdMap.set(sessionId, await hashThreadId(sessionId));
|
|
164
|
+
sessionCount++;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
console.log(chalk.gray(`Found ${sessionCount} unique session IDs to hash\n`));
|
|
171
|
+
|
|
172
|
+
// Show some examples
|
|
173
|
+
console.log(chalk.gray('Example mappings:'));
|
|
174
|
+
let exampleCount = 0;
|
|
175
|
+
for (const [original, hashed] of threadIdMap) {
|
|
176
|
+
if (exampleCount >= 5) break;
|
|
177
|
+
console.log(chalk.gray(` ${original} ā ${hashed}`));
|
|
178
|
+
exampleCount++;
|
|
179
|
+
}
|
|
180
|
+
console.log('');
|
|
181
|
+
|
|
182
|
+
// Find prepared data directory
|
|
183
|
+
const preparedDir = join(preparedDataDir, dataset, memoryConfig);
|
|
184
|
+
|
|
185
|
+
if (!existsSync(preparedDir)) {
|
|
186
|
+
console.error(chalk.red(`No prepared data found at: ${preparedDir}`));
|
|
187
|
+
console.error(chalk.gray(`Run 'longmemeval prepare' first`));
|
|
188
|
+
process.exit(1);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// Get all question directories
|
|
192
|
+
const questionDirs = await readdir(preparedDir);
|
|
193
|
+
|
|
194
|
+
let filesModified = 0;
|
|
195
|
+
let totalReplacements = 0;
|
|
196
|
+
let filesSkipped = 0;
|
|
197
|
+
|
|
198
|
+
for (const questionDir of questionDirs) {
|
|
199
|
+
const omPath = join(preparedDir, questionDir, 'om.json');
|
|
200
|
+
|
|
201
|
+
if (!existsSync(omPath)) {
|
|
202
|
+
filesSkipped++;
|
|
203
|
+
continue;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
try {
|
|
207
|
+
const content = await readFile(omPath, 'utf-8');
|
|
208
|
+
const { content: newContent, replacements } = processOmJson(content, threadIdMap);
|
|
209
|
+
|
|
210
|
+
if (replacements > 0) {
|
|
211
|
+
if (!dryRun) {
|
|
212
|
+
await writeFile(omPath, newContent, 'utf-8');
|
|
213
|
+
}
|
|
214
|
+
filesModified++;
|
|
215
|
+
totalReplacements += replacements;
|
|
216
|
+
console.log(chalk.green(` ā ${questionDir}: ${replacements} thread ID(s) replaced`));
|
|
217
|
+
} else {
|
|
218
|
+
filesSkipped++;
|
|
219
|
+
}
|
|
220
|
+
} catch (error) {
|
|
221
|
+
console.error(chalk.red(` ā ${questionDir}: Error processing file`));
|
|
222
|
+
console.error(chalk.gray(` ${error}`));
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// Summary
|
|
227
|
+
console.log('');
|
|
228
|
+
console.log(chalk.bold('Summary:'));
|
|
229
|
+
console.log(chalk.gray(` Files modified: ${filesModified}`));
|
|
230
|
+
console.log(chalk.gray(` Total replacements: ${totalReplacements}`));
|
|
231
|
+
console.log(chalk.gray(` Files skipped: ${filesSkipped}`));
|
|
232
|
+
|
|
233
|
+
if (dryRun && filesModified > 0) {
|
|
234
|
+
console.log(chalk.yellow(`\nā ļø Run without --dry-run to apply changes`));
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
}
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import { existsSync } from 'fs';
|
|
2
|
+
import { readFile, readdir } from 'fs/promises';
|
|
3
|
+
import { join } from 'path';
|
|
4
|
+
import chalk from 'chalk';
|
|
5
|
+
|
|
6
|
+
interface PartialResult {
|
|
7
|
+
question_id: string;
|
|
8
|
+
question_type: string;
|
|
9
|
+
is_correct: boolean;
|
|
10
|
+
improved_is_correct?: boolean;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
interface TypeStats {
|
|
14
|
+
correct: number;
|
|
15
|
+
fixed: number;
|
|
16
|
+
total: number;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface PartialResultsOptions {
|
|
20
|
+
config?: string;
|
|
21
|
+
runId?: string;
|
|
22
|
+
outputDir: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export class PartialResultsCommand {
|
|
26
|
+
async run(options: PartialResultsOptions): Promise<void> {
|
|
27
|
+
const resultsDir = options.outputDir;
|
|
28
|
+
|
|
29
|
+
// If no config specified, list all configs with partial results
|
|
30
|
+
if (!options.config) {
|
|
31
|
+
await this.listAllPartialResults(resultsDir);
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const configDir = join(resultsDir, options.config);
|
|
36
|
+
if (!existsSync(configDir)) {
|
|
37
|
+
console.log(chalk.red(`No results found for config: ${options.config}`));
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Find the run to analyze
|
|
42
|
+
let runDir: string;
|
|
43
|
+
if (options.runId) {
|
|
44
|
+
runDir = join(configDir, options.runId);
|
|
45
|
+
if (!existsSync(runDir)) {
|
|
46
|
+
console.log(chalk.red(`Run not found: ${options.runId}`));
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
} else {
|
|
50
|
+
// Find the most recent run
|
|
51
|
+
const runs = await readdir(configDir);
|
|
52
|
+
const runDirs = runs.filter(r => r.startsWith('run_')).sort();
|
|
53
|
+
if (runDirs.length === 0) {
|
|
54
|
+
console.log(chalk.red(`No runs found for config: ${options.config}`));
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
runDir = join(configDir, runDirs[runDirs.length - 1]);
|
|
58
|
+
console.log(chalk.gray(`Using most recent run: ${runDirs[runDirs.length - 1]}\n`));
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Check for results.jsonl
|
|
62
|
+
const resultsFile = join(runDir, 'results.jsonl');
|
|
63
|
+
if (!existsSync(resultsFile)) {
|
|
64
|
+
console.log(chalk.red(`No results.jsonl found in ${runDir}`));
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// Check if this is a completed run (has metrics.json)
|
|
69
|
+
const metricsFile = join(runDir, 'metrics.json');
|
|
70
|
+
const isComplete = existsSync(metricsFile);
|
|
71
|
+
|
|
72
|
+
// Load and analyze results
|
|
73
|
+
const results = await this.loadResults(resultsFile);
|
|
74
|
+
this.displayResults(options.config, results, isComplete);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async loadResults(resultsFile: string): Promise<PartialResult[]> {
|
|
78
|
+
const content = await readFile(resultsFile, 'utf-8');
|
|
79
|
+
return content
|
|
80
|
+
.trim()
|
|
81
|
+
.split('\n')
|
|
82
|
+
.filter(line => line.trim())
|
|
83
|
+
.map(line => JSON.parse(line));
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
displayResults(config: string, results: PartialResult[], isComplete: boolean): void {
|
|
87
|
+
const total = results.length;
|
|
88
|
+
const correct = results.filter(r => r.is_correct).length;
|
|
89
|
+
const fixedCorrect = results.filter(r => r.is_correct || r.improved_is_correct).length;
|
|
90
|
+
|
|
91
|
+
const status = isComplete ? chalk.green('(complete)') : chalk.yellow('(partial)');
|
|
92
|
+
|
|
93
|
+
console.log(chalk.bold(`\nš Results for ${chalk.cyan(config)} ${status}\n`));
|
|
94
|
+
console.log(`Progress: ${chalk.bold(total)}/500 questions (${(total / 5).toFixed(1)}%)`);
|
|
95
|
+
console.log();
|
|
96
|
+
console.log(`Overall Accuracy: ${correct}/${total} = ${chalk.bold(((100 * correct) / total).toFixed(2) + '%')}`);
|
|
97
|
+
console.log(
|
|
98
|
+
`Fixed Accuracy: ${fixedCorrect}/${total} = ${chalk.bold(((100 * fixedCorrect) / total).toFixed(2) + '%')}`,
|
|
99
|
+
);
|
|
100
|
+
console.log();
|
|
101
|
+
|
|
102
|
+
// By type
|
|
103
|
+
const byType: Record<string, TypeStats> = {};
|
|
104
|
+
for (const r of results) {
|
|
105
|
+
const t = r.question_type || 'unknown';
|
|
106
|
+
if (!byType[t]) byType[t] = { correct: 0, fixed: 0, total: 0 };
|
|
107
|
+
byType[t].total++;
|
|
108
|
+
if (r.is_correct) byType[t].correct++;
|
|
109
|
+
if (r.is_correct || r.improved_is_correct) byType[t].fixed++;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
console.log(chalk.bold('By Question Type:'));
|
|
113
|
+
console.log(chalk.gray('ā'.repeat(70)));
|
|
114
|
+
console.log(
|
|
115
|
+
chalk.gray('Type'.padEnd(32)),
|
|
116
|
+
chalk.gray('Count'.padEnd(8)),
|
|
117
|
+
chalk.gray('Accuracy'.padEnd(12)),
|
|
118
|
+
chalk.gray('Fixed'),
|
|
119
|
+
);
|
|
120
|
+
console.log(chalk.gray('ā'.repeat(70)));
|
|
121
|
+
|
|
122
|
+
for (const [t, stats] of Object.entries(byType).sort((a, b) => a[0].localeCompare(b[0]))) {
|
|
123
|
+
const acc = stats.total > 0 ? (100 * stats.correct) / stats.total : 0;
|
|
124
|
+
const fix = stats.total > 0 ? (100 * stats.fixed) / stats.total : 0;
|
|
125
|
+
console.log(
|
|
126
|
+
t.padEnd(32),
|
|
127
|
+
String(stats.total).padEnd(8),
|
|
128
|
+
(acc.toFixed(1) + '%').padStart(6).padEnd(12),
|
|
129
|
+
(fix.toFixed(1) + '%').padStart(6),
|
|
130
|
+
);
|
|
131
|
+
}
|
|
132
|
+
console.log();
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
async listAllPartialResults(resultsDir: string): Promise<void> {
|
|
136
|
+
if (!existsSync(resultsDir)) {
|
|
137
|
+
console.log(chalk.red(`Results directory not found: ${resultsDir}`));
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const configs = await readdir(resultsDir);
|
|
142
|
+
const configsWithResults: Array<{ config: string; runId: string; count: number; isComplete: boolean }> = [];
|
|
143
|
+
|
|
144
|
+
for (const config of configs) {
|
|
145
|
+
const configDir = join(resultsDir, config);
|
|
146
|
+
try {
|
|
147
|
+
const runs = await readdir(configDir);
|
|
148
|
+
const runDirs = runs.filter(r => r.startsWith('run_')).sort();
|
|
149
|
+
if (runDirs.length > 0) {
|
|
150
|
+
const latestRun = runDirs[runDirs.length - 1];
|
|
151
|
+
const resultsFile = join(configDir, latestRun, 'results.jsonl');
|
|
152
|
+
const metricsFile = join(configDir, latestRun, 'metrics.json');
|
|
153
|
+
if (existsSync(resultsFile)) {
|
|
154
|
+
const content = await readFile(resultsFile, 'utf-8');
|
|
155
|
+
const count = content
|
|
156
|
+
.trim()
|
|
157
|
+
.split('\n')
|
|
158
|
+
.filter(l => l.trim()).length;
|
|
159
|
+
configsWithResults.push({
|
|
160
|
+
config,
|
|
161
|
+
runId: latestRun,
|
|
162
|
+
count,
|
|
163
|
+
isComplete: existsSync(metricsFile),
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
} catch {
|
|
168
|
+
// Skip non-directories
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
if (configsWithResults.length === 0) {
|
|
173
|
+
console.log(chalk.yellow('No results found.'));
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
console.log(chalk.bold('\nš Available Results:\n'));
|
|
178
|
+
console.log(chalk.gray('ā'.repeat(80)));
|
|
179
|
+
console.log(
|
|
180
|
+
chalk.gray('Config'.padEnd(40)),
|
|
181
|
+
chalk.gray('Run'.padEnd(24)),
|
|
182
|
+
chalk.gray('Questions'.padEnd(12)),
|
|
183
|
+
chalk.gray('Status'),
|
|
184
|
+
);
|
|
185
|
+
console.log(chalk.gray('ā'.repeat(80)));
|
|
186
|
+
|
|
187
|
+
for (const { config, runId, count, isComplete } of configsWithResults.sort((a, b) =>
|
|
188
|
+
a.config.localeCompare(b.config),
|
|
189
|
+
)) {
|
|
190
|
+
const status = isComplete ? chalk.green('complete') : chalk.yellow('partial');
|
|
191
|
+
console.log(config.padEnd(40), runId.padEnd(24), String(count).padEnd(12), status);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
console.log();
|
|
195
|
+
console.log(chalk.gray('Usage: pnpm run partial <config> [--run-id <id>]'));
|
|
196
|
+
console.log();
|
|
197
|
+
}
|
|
198
|
+
}
|