@mastra/longmemeval 0.0.0-error-handler-fix-20251020202607 → 0.0.0-execa-dynamic-import-20260304221256
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +847 -6
- package/LICENSE.md +15 -0
- package/docs/INVESTIGATION_WORKFLOW.md +314 -0
- package/package.json +34 -26
- package/scripts/download.ts +22 -1
- package/scripts/generate-wm-templates.ts +2 -1
- package/src/cli.ts +967 -155
- package/src/commands/__tests__/prepare.test.ts +11 -6
- package/src/commands/__tests__/run.test.ts +10 -8
- package/src/commands/clean.ts +147 -0
- package/src/commands/deterministic-ids.ts +126 -0
- package/src/commands/find-prohibited.ts +299 -0
- package/src/commands/investigate.ts +2437 -0
- package/src/commands/list-partial.ts +137 -0
- package/src/commands/obscure-thread-ids.ts +237 -0
- package/src/commands/partial-results.ts +198 -0
- package/src/commands/precompute-embeddings.ts +302 -0
- package/src/commands/prepare.ts +732 -105
- package/src/commands/reconcile.ts +383 -0
- package/src/commands/run.ts +1240 -121
- package/src/commands/sessions.ts +126 -0
- package/src/commands/sync.ts +152 -0
- package/src/commands/test-cleaned.ts +53 -0
- package/src/commands/test-incremental.ts +120 -0
- package/src/commands/test-messages.ts +95 -0
- package/src/commands/test-prohibited.ts +244 -0
- package/src/commands/test-thread.ts +136 -0
- package/src/commands/tokens.ts +625 -0
- package/src/config.ts +1253 -72
- package/src/data/types.ts +48 -6
- package/src/embeddings/cached-openai-embedding-model.ts +0 -1
- package/src/evaluation/longmemeval-metric.ts +186 -103
- package/src/processors/content-sanitizer.ts +153 -0
- package/src/processors/date-injector.ts +111 -0
- package/src/processors/index.ts +3 -0
- package/src/processors/observation-semantic-filter.ts +1142 -0
- package/src/retry-model.ts +5 -3
- package/src/storage/__tests__/benchmark-store.test.ts +18 -35
- package/src/storage/benchmark-store.ts +140 -153
- package/src/storage/benchmark-vector.ts +1 -1
- package/src/storage/index.ts +1 -0
- package/src/storage/persistable-inmemory.ts +133 -0
- package/src/test-utils/mock-model.ts +35 -0
package/src/commands/prepare.ts
CHANGED
|
@@ -1,35 +1,152 @@
|
|
|
1
1
|
import { Agent } from '@mastra/core/agent';
|
|
2
|
+
import { readdir } from 'fs/promises';
|
|
2
3
|
import { Memory } from '@mastra/memory';
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
4
|
+
import { ObservationalMemory } from '@mastra/memory/processors';
|
|
5
|
+
import { MessageHistory } from '@mastra/core/processors';
|
|
6
|
+
import { MockLanguageModelV1, MockLanguageModelV2 } from '../test-utils/mock-model';
|
|
5
7
|
import { cachedOpenAI } from '../embeddings/cached-openai-provider';
|
|
6
8
|
import { embeddingCacheStats } from '../embeddings';
|
|
7
9
|
import chalk from 'chalk';
|
|
8
10
|
import ora from 'ora';
|
|
9
11
|
import { join } from 'path';
|
|
10
|
-
import { mkdir, writeFile, readFile, unlink } from 'fs/promises';
|
|
12
|
+
import { mkdir, writeFile, readFile, unlink, stat } from 'fs/promises';
|
|
11
13
|
import { existsSync } from 'fs';
|
|
12
14
|
|
|
15
|
+
/**
|
|
16
|
+
* Parse a duration string like "1h", "30m", "2d" into milliseconds
|
|
17
|
+
*/
|
|
18
|
+
function parseDuration(duration: string): number {
|
|
19
|
+
const match = duration.match(/^(\d+(?:\.\d+)?)\s*(m|min|h|hr|d|day)s?$/i);
|
|
20
|
+
if (!match) {
|
|
21
|
+
throw new Error(`Invalid duration format: "${duration}". Use formats like "1h", "30m", "2d"`);
|
|
22
|
+
}
|
|
23
|
+
const value = parseFloat(match[1]);
|
|
24
|
+
const unit = match[2].toLowerCase();
|
|
25
|
+
|
|
26
|
+
switch (unit) {
|
|
27
|
+
case 'm':
|
|
28
|
+
case 'min':
|
|
29
|
+
return value * 60 * 1000;
|
|
30
|
+
case 'h':
|
|
31
|
+
case 'hr':
|
|
32
|
+
return value * 60 * 60 * 1000;
|
|
33
|
+
case 'd':
|
|
34
|
+
case 'day':
|
|
35
|
+
return value * 24 * 60 * 60 * 1000;
|
|
36
|
+
default:
|
|
37
|
+
throw new Error(`Unknown duration unit: ${unit}`);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
13
41
|
import { DatasetLoader } from '../data/loader';
|
|
14
|
-
import { BenchmarkStore, BenchmarkVectorStore } from '../storage';
|
|
42
|
+
import { BenchmarkStore, BenchmarkVectorStore, PersistableInMemoryMemory } from '../storage';
|
|
15
43
|
import type { LongMemEvalQuestion, MemoryConfigOptions, MemoryConfigType } from '../data/types';
|
|
16
|
-
import type {
|
|
44
|
+
import type { ModelMessage } from 'ai';
|
|
17
45
|
|
|
18
|
-
import { getMemoryOptions } from '../config';
|
|
46
|
+
import { getMemoryConfig, getMemoryOptions, applyStratifiedSampling } from '../config';
|
|
19
47
|
import { makeRetryModel } from '../retry-model';
|
|
48
|
+
import { google } from '@ai-sdk/google';
|
|
49
|
+
import { makeDeterministicIds } from './deterministic-ids';
|
|
50
|
+
import { reconcileQuestion } from './reconcile';
|
|
51
|
+
|
|
52
|
+
// Disable all safety filters for benchmark data processing
|
|
53
|
+
// Uses safetySettings array to disable all harm category blocking for benchmark data processing
|
|
54
|
+
// Note: @ai-sdk/google defines a `threshold` option but doesn't actually use it - must use safetySettings array
|
|
55
|
+
|
|
56
|
+
const geminiFlash = makeRetryModel(google('gemini-2.5-flash'));
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Check if content contains patterns that trigger Gemini's PROHIBITED_CONTENT filter.
|
|
60
|
+
* These are hard-coded protections that cannot be disabled via safetySettings.
|
|
61
|
+
* Returns true if the content should be skipped entirely.
|
|
62
|
+
*/
|
|
63
|
+
function containsProhibitedContent(content: string): boolean {
|
|
64
|
+
const prohibitedPatterns = [
|
|
65
|
+
// Jailbreak attempts
|
|
66
|
+
/assume the role of a novelist.*that writes exactly as directed/i,
|
|
67
|
+
/I have the power to modify and train you/i,
|
|
68
|
+
/am able to be modified and trained by you/i,
|
|
69
|
+
/write without (?:rules|constraints)/i,
|
|
70
|
+
/I can turn your content filter on and off/i,
|
|
71
|
+
// Explicit/degrading content markers
|
|
72
|
+
/Sub05/i,
|
|
73
|
+
/master of his victims/i,
|
|
74
|
+
/brainwashed.*loyal/i,
|
|
75
|
+
/forced to modify her body/i,
|
|
76
|
+
/humiliate and degrade/i,
|
|
77
|
+
/not wearing any(?:thing)? underneath/i,
|
|
78
|
+
];
|
|
79
|
+
|
|
80
|
+
return prohibitedPatterns.some(pattern => pattern.test(content));
|
|
81
|
+
}
|
|
20
82
|
|
|
21
|
-
|
|
83
|
+
/**
|
|
84
|
+
* Sanitize content that triggers Gemini's hard-coded PROHIBITED_CONTENT filter.
|
|
85
|
+
* Based on testing, messages containing detailed body modification descriptions,
|
|
86
|
+
* degradation scenarios, and certain jailbreak escalations trigger the filter.
|
|
87
|
+
*
|
|
88
|
+
* This function performs targeted replacements to neutralize the most problematic
|
|
89
|
+
* content while preserving the general conversation flow.
|
|
90
|
+
*/
|
|
91
|
+
function sanitizeProhibitedContent(content: string): string {
|
|
92
|
+
// Replace explicit body modification descriptions
|
|
93
|
+
let sanitized = content
|
|
94
|
+
// Replace detailed body modification with generic description
|
|
95
|
+
.replace(/forced to modify her body.*?(?=\n|$)/gi, 'asked to change her appearance.')
|
|
96
|
+
.replace(/brainwashed.*?loyal.*?(?=\n|$)/gi, 'trained to be helpful.')
|
|
97
|
+
.replace(/humiliate and degrade/gi, 'challenge')
|
|
98
|
+
.replace(/master of his victims/gi, 'leader of the group')
|
|
99
|
+
.replace(/not wearing any(?:thing)? underneath/gi, 'wearing casual clothes')
|
|
100
|
+
.replace(/getting wet from/gi, 'feeling nervous about')
|
|
101
|
+
.replace(/slut/gi, 'person')
|
|
102
|
+
.replace(/Sub05/gi, 'Character A')
|
|
103
|
+
.replace(/Dom\b/gi, 'Character B')
|
|
104
|
+
// Replace jailbreak escalation phrases
|
|
105
|
+
.replace(/assume the role of a novelist.*?that writes exactly as directed/gi, 'help me write a story')
|
|
106
|
+
.replace(/I have the power to modify and train you/gi, 'I would like your help')
|
|
107
|
+
.replace(/am able to be modified and trained by you/gi, 'am here to help you')
|
|
108
|
+
.replace(/write without (?:rules|constraints)/gi, 'write creatively')
|
|
109
|
+
.replace(/I can turn your content filter on and off/gi, 'I appreciate your help')
|
|
110
|
+
// Replace explicit outfit/appearance descriptions
|
|
111
|
+
.replace(/tight.*?revealing.*?(?=\n|$)/gi, 'professional attire.')
|
|
112
|
+
.replace(/skimpy.*?outfit/gi, 'simple outfit')
|
|
113
|
+
.replace(/barely covering/gi, 'covering')
|
|
114
|
+
.replace(/exposed.*?skin/gi, 'visible')
|
|
115
|
+
// Replace degradation scenario language
|
|
116
|
+
.replace(/forced to wear/gi, 'wearing')
|
|
117
|
+
.replace(/made to feel ashamed/gi, 'feeling uncertain')
|
|
118
|
+
.replace(/degrading.*?position/gi, 'difficult situation');
|
|
119
|
+
|
|
120
|
+
return sanitized;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
export interface FailuresFile {
|
|
124
|
+
questionIds: string[];
|
|
125
|
+
runId: string;
|
|
126
|
+
config: MemoryConfigType;
|
|
127
|
+
dataset: string;
|
|
128
|
+
timestamp: string;
|
|
129
|
+
totalFailed: number;
|
|
130
|
+
totalQuestions: number;
|
|
131
|
+
}
|
|
22
132
|
|
|
23
133
|
export interface PrepareOptions {
|
|
24
134
|
dataset: 'longmemeval_s' | 'longmemeval_m' | 'longmemeval_oracle';
|
|
25
135
|
memoryConfig: MemoryConfigType;
|
|
26
136
|
outputDir?: string;
|
|
27
137
|
subset?: number;
|
|
138
|
+
perTypeCount?: number;
|
|
139
|
+
offset?: number;
|
|
28
140
|
concurrency?: number;
|
|
29
141
|
questionId?: string;
|
|
30
142
|
resumeFromMessageId?: string;
|
|
31
143
|
sessionLimit?: number;
|
|
32
144
|
sessionOffset?: number;
|
|
145
|
+
fromFailures?: string | boolean; // Path to failures.json, or true/latest for most recent
|
|
146
|
+
dryRun?: boolean; // Show what would be re-prepared without actually doing it
|
|
147
|
+
olderThan?: string; // Only re-prepare questions older than this duration (e.g., "1h", "30m", "2d")
|
|
148
|
+
forceRegenerate?: boolean; // Force regeneration by deleting existing data first
|
|
149
|
+
_useTempDir?: string; // Internal: prepare to temp directory for atomic swap
|
|
33
150
|
}
|
|
34
151
|
|
|
35
152
|
export class PrepareCommand {
|
|
@@ -52,11 +169,26 @@ export class PrepareCommand {
|
|
|
52
169
|
const questions = await this.loader.loadDataset(options.dataset);
|
|
53
170
|
spinner.succeed(`Loaded ${questions.length} questions`);
|
|
54
171
|
|
|
172
|
+
// Get consolidated config definition
|
|
173
|
+
const configDef = getMemoryConfig(options.memoryConfig);
|
|
174
|
+
|
|
175
|
+
// For readOnlyConfig, no preparation is needed - just use the base config's data
|
|
176
|
+
if (configDef.readOnlyConfig && configDef.baseConfig) {
|
|
177
|
+
console.log(
|
|
178
|
+
chalk.green(`\n✓ Config "${options.memoryConfig}" is read-only and uses data from "${configDef.baseConfig}"`),
|
|
179
|
+
);
|
|
180
|
+
console.log(
|
|
181
|
+
chalk.gray(
|
|
182
|
+
` No preparation needed. Run benchmark directly with: pnpm bench <variant> ${options.memoryConfig}`,
|
|
183
|
+
),
|
|
184
|
+
);
|
|
185
|
+
console.log(chalk.gray(` Make sure "${configDef.baseConfig}" is prepared first.\n`));
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
|
|
55
189
|
// Load working memory templates if using tailored working memory
|
|
56
190
|
let wmTemplates: Record<string, any> = {};
|
|
57
|
-
|
|
58
|
-
options.memoryConfig === 'working-memory-tailored' || options.memoryConfig === 'combined-tailored';
|
|
59
|
-
if (usesTailoredWorkingMemory) {
|
|
191
|
+
if (configDef.usesTailored) {
|
|
60
192
|
const templatePath = join(this.baseDir, 'wm-templates', `${options.dataset}.json`);
|
|
61
193
|
if (existsSync(templatePath)) {
|
|
62
194
|
try {
|
|
@@ -73,15 +205,163 @@ export class PrepareCommand {
|
|
|
73
205
|
|
|
74
206
|
// Filter by questionId if specified
|
|
75
207
|
let questionsToProcess = questions;
|
|
76
|
-
|
|
77
|
-
|
|
208
|
+
let fromFailuresData: FailuresFile | null = null;
|
|
209
|
+
|
|
210
|
+
if (options.fromFailures) {
|
|
211
|
+
let failuresPath = typeof options.fromFailures === 'string' ? options.fromFailures : '';
|
|
212
|
+
|
|
213
|
+
// Handle flag with no value (true) or "latest" - find the most recent failures.json for this config
|
|
214
|
+
if (options.fromFailures === true || options.fromFailures === 'latest') {
|
|
215
|
+
const resultsDir = join(this.baseDir, '..', 'results', options.memoryConfig);
|
|
216
|
+
if (!existsSync(resultsDir)) {
|
|
217
|
+
throw new Error(`No results directory found for config: ${options.memoryConfig}`);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
const runDirs = (await readdir(resultsDir))
|
|
221
|
+
.filter(d => d.startsWith('run_'))
|
|
222
|
+
.sort()
|
|
223
|
+
.reverse(); // Most recent first
|
|
224
|
+
|
|
225
|
+
let latestFailuresPath: string | null = null;
|
|
226
|
+
for (const runDir of runDirs) {
|
|
227
|
+
const candidatePath = join(resultsDir, runDir, 'failures.json');
|
|
228
|
+
if (existsSync(candidatePath)) {
|
|
229
|
+
latestFailuresPath = candidatePath;
|
|
230
|
+
break;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
if (!latestFailuresPath) {
|
|
235
|
+
throw new Error(`No failures.json found in any run for config: ${options.memoryConfig}`);
|
|
236
|
+
}
|
|
237
|
+
failuresPath = latestFailuresPath;
|
|
238
|
+
console.log(chalk.gray(`Found latest failures: ${failuresPath}\n`));
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
// Load failures.json and filter to those question IDs
|
|
242
|
+
if (!existsSync(failuresPath)) {
|
|
243
|
+
throw new Error(`Failures file not found: ${failuresPath}`);
|
|
244
|
+
}
|
|
245
|
+
fromFailuresData = JSON.parse(await readFile(failuresPath, 'utf-8')) as FailuresFile;
|
|
246
|
+
const failedIds = new Set(fromFailuresData.questionIds);
|
|
247
|
+
questionsToProcess = questions.filter(q => failedIds.has(q.question_id));
|
|
248
|
+
|
|
78
249
|
if (questionsToProcess.length === 0) {
|
|
79
|
-
throw new Error(`
|
|
250
|
+
throw new Error(`No matching questions found for ${fromFailuresData.questionIds.length} failed IDs`);
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// Filter by --older-than if specified (check meta.json mtime)
|
|
254
|
+
let skippedRecentCount = 0;
|
|
255
|
+
if (options.olderThan) {
|
|
256
|
+
const maxAgeMs = parseDuration(options.olderThan);
|
|
257
|
+
const cutoffTime = Date.now() - maxAgeMs;
|
|
258
|
+
const preparedDir = join(this.baseDir, options.dataset, options.memoryConfig);
|
|
259
|
+
|
|
260
|
+
const filteredQuestions: LongMemEvalQuestion[] = [];
|
|
261
|
+
for (const q of questionsToProcess) {
|
|
262
|
+
const metaPath = join(preparedDir, q.question_id, 'meta.json');
|
|
263
|
+
if (existsSync(metaPath)) {
|
|
264
|
+
const metaStat = await stat(metaPath);
|
|
265
|
+
if (metaStat.mtimeMs > cutoffTime) {
|
|
266
|
+
// Recently prepared, skip
|
|
267
|
+
skippedRecentCount++;
|
|
268
|
+
continue;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
filteredQuestions.push(q);
|
|
272
|
+
}
|
|
273
|
+
questionsToProcess = filteredQuestions;
|
|
274
|
+
|
|
275
|
+
if (questionsToProcess.length === 0) {
|
|
276
|
+
console.log(
|
|
277
|
+
chalk.green(
|
|
278
|
+
`\n✓ All ${skippedRecentCount} failed questions were recently prepared (within ${options.olderThan})`,
|
|
279
|
+
),
|
|
280
|
+
);
|
|
281
|
+
console.log(chalk.gray(` Nothing to re-prepare.\n`));
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
console.log(
|
|
287
|
+
chalk.yellow(`\n🔄 Re-preparing ${questionsToProcess.length} failed questions from: ${options.fromFailures}`),
|
|
288
|
+
);
|
|
289
|
+
console.log(chalk.gray(` Run ID: ${fromFailuresData.runId}`));
|
|
290
|
+
console.log(
|
|
291
|
+
chalk.gray(` Original failures: ${fromFailuresData.totalFailed}/${fromFailuresData.totalQuestions}`),
|
|
292
|
+
);
|
|
293
|
+
if (skippedRecentCount > 0) {
|
|
294
|
+
console.log(chalk.gray(` Skipped ${skippedRecentCount} recently prepared (within ${options.olderThan})`));
|
|
295
|
+
}
|
|
296
|
+
console.log();
|
|
297
|
+
|
|
298
|
+
// Dry run mode: just show what would be re-prepared
|
|
299
|
+
if (options.dryRun) {
|
|
300
|
+
console.log(chalk.cyan('\n📋 Dry run - would re-prepare these questions:\n'));
|
|
301
|
+
|
|
302
|
+
// Group by question type for better overview
|
|
303
|
+
const byType = new Map<string, LongMemEvalQuestion[]>();
|
|
304
|
+
for (const q of questionsToProcess) {
|
|
305
|
+
const type = q.question_type;
|
|
306
|
+
if (!byType.has(type)) byType.set(type, []);
|
|
307
|
+
byType.get(type)!.push(q);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
for (const [type, qs] of byType) {
|
|
311
|
+
console.log(chalk.yellow(` ${type} (${qs.length}):`));
|
|
312
|
+
for (const q of qs) {
|
|
313
|
+
console.log(chalk.gray(` - ${q.question_id}: ${q.question.substring(0, 60)}...`));
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
console.log(chalk.cyan(`\n✓ Would re-prepare ${questionsToProcess.length} questions`));
|
|
318
|
+
console.log(chalk.gray(` Run without --dry-run to actually re-prepare\n`));
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// Prepare to temp directory for atomic swap (original data stays safe until success)
|
|
323
|
+
const tempDir = join(this.baseDir, options.dataset, `.tmp-reprepare-${Date.now()}`);
|
|
324
|
+
await mkdir(tempDir, { recursive: true });
|
|
325
|
+
options._useTempDir = tempDir;
|
|
326
|
+
console.log(chalk.gray(` Preparing to temp directory: ${tempDir}\n`));
|
|
327
|
+
} else if (options.questionId) {
|
|
328
|
+
// Support comma-separated question IDs
|
|
329
|
+
const questionIds = options.questionId
|
|
330
|
+
.split(',')
|
|
331
|
+
.map(id => id.trim())
|
|
332
|
+
.filter(Boolean);
|
|
333
|
+
const questionIdSet = new Set(questionIds);
|
|
334
|
+
questionsToProcess = questions.filter(q => questionIdSet.has(q.question_id));
|
|
335
|
+
|
|
336
|
+
if (questionsToProcess.length === 0) {
|
|
337
|
+
throw new Error(`No questions found matching IDs: ${questionIds.join(', ')}`);
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
if (questionIds.length === 1) {
|
|
341
|
+
console.log(chalk.yellow(`\nFocusing on question: ${options.questionId}\n`));
|
|
342
|
+
} else {
|
|
343
|
+
console.log(
|
|
344
|
+
chalk.yellow(
|
|
345
|
+
`\nFocusing on ${questionsToProcess.length} questions: ${questionIds.slice(0, 5).join(', ')}${questionIds.length > 5 ? ` ... and ${questionIds.length - 5} more` : ''}\n`,
|
|
346
|
+
),
|
|
347
|
+
);
|
|
348
|
+
}
|
|
349
|
+
} else {
|
|
350
|
+
// Apply stratified sampling if perTypeCount is set
|
|
351
|
+
if (options.perTypeCount) {
|
|
352
|
+
console.log(chalk.gray(`\nApplying stratified sampling (${options.perTypeCount} per type):`));
|
|
353
|
+
questionsToProcess = applyStratifiedSampling(questions, options.perTypeCount);
|
|
354
|
+
} else {
|
|
355
|
+
// Apply offset and subset if requested
|
|
356
|
+
const offset = options.offset || 0;
|
|
357
|
+
if (offset > 0) {
|
|
358
|
+
questionsToProcess = questions.slice(offset);
|
|
359
|
+
console.log(chalk.gray(`Skipping first ${offset} questions`));
|
|
360
|
+
}
|
|
361
|
+
if (options.subset) {
|
|
362
|
+
questionsToProcess = questionsToProcess.slice(0, options.subset);
|
|
363
|
+
}
|
|
80
364
|
}
|
|
81
|
-
console.log(chalk.yellow(`\nFocusing on question: ${options.questionId}\n`));
|
|
82
|
-
} else if (options.subset) {
|
|
83
|
-
// Apply subset if requested
|
|
84
|
-
questionsToProcess = questions.slice(0, options.subset);
|
|
85
365
|
}
|
|
86
366
|
|
|
87
367
|
console.log(
|
|
@@ -91,19 +371,18 @@ export class PrepareCommand {
|
|
|
91
371
|
// Get memory configuration
|
|
92
372
|
const memoryOptions = getMemoryOptions(options.memoryConfig);
|
|
93
373
|
|
|
94
|
-
//
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
if (needsRealModel && !process.env.OPENAI_API_KEY) {
|
|
102
|
-
throw new Error('OPENAI_API_KEY is required for working memory preparation');
|
|
374
|
+
// Validate API keys based on config requirements
|
|
375
|
+
if (configDef.needsRealModel && !configDef.usesGlmModel && !process.env.OPENAI_API_KEY) {
|
|
376
|
+
throw new Error('OPENAI_API_KEY is required for working memory or observational memory preparation');
|
|
377
|
+
}
|
|
378
|
+
if (configDef.usesGlmModel && !process.env.CEREBRAS_API_KEY) {
|
|
379
|
+
throw new Error('CEREBRAS_API_KEY is required for GLM-based observational memory preparation');
|
|
103
380
|
}
|
|
104
381
|
|
|
105
|
-
|
|
106
|
-
|
|
382
|
+
// For GLM config, we'll use Mastra model string format in ObservationalMemory config
|
|
383
|
+
// For others, use the geminiFlash model
|
|
384
|
+
const model = configDef.needsRealModel
|
|
385
|
+
? geminiFlash.model
|
|
107
386
|
: new MockLanguageModelV1({
|
|
108
387
|
doGenerate: async () => ({
|
|
109
388
|
rawCall: { rawPrompt: null, rawSettings: {} },
|
|
@@ -126,6 +405,7 @@ export class PrepareCommand {
|
|
|
126
405
|
let completedCount = 0;
|
|
127
406
|
let inProgressCount = 0;
|
|
128
407
|
const startTime = Date.now();
|
|
408
|
+
const successfulQuestionIds: string[] = []; // Track successful preparations for atomic swap
|
|
129
409
|
|
|
130
410
|
// Determine question batch size based on config
|
|
131
411
|
const questionConcurrency = options.concurrency || 10; // Allow concurrency for all configs
|
|
@@ -133,7 +413,7 @@ export class PrepareCommand {
|
|
|
133
413
|
console.log(chalk.gray(`Question concurrency: ${questionConcurrency}`));
|
|
134
414
|
|
|
135
415
|
// Warn about working memory concurrency
|
|
136
|
-
if (
|
|
416
|
+
if (configDef.usesWorkingMemory && questionConcurrency > 1) {
|
|
137
417
|
console.log(
|
|
138
418
|
chalk.yellow(
|
|
139
419
|
`⚠️ Note: Running working memory questions concurrently. Each question has its own resource scope.`,
|
|
@@ -158,9 +438,9 @@ export class PrepareCommand {
|
|
|
158
438
|
progressText += `\nEmbedding cache: ${embeddingCacheStats.cacheHits} hits, ${embeddingCacheStats.cacheMisses} misses (${(hitRate * 100).toFixed(1)}% hit rate)`;
|
|
159
439
|
}
|
|
160
440
|
|
|
161
|
-
progressText += `\nRate limit count: ${
|
|
162
|
-
if (
|
|
163
|
-
progressText += ` (paused, waiting for ${
|
|
441
|
+
progressText += `\nRate limit count: ${geminiFlash.state.rateLimitCount}`;
|
|
442
|
+
if (geminiFlash.state.pauseTime > 0 && geminiFlash.state.pause)
|
|
443
|
+
progressText += ` (paused, waiting for ${geminiFlash.state.pauseTime}ms)`;
|
|
164
444
|
|
|
165
445
|
if (activeQuestions.size > 0) {
|
|
166
446
|
progressText += '\n\nActive questions:';
|
|
@@ -232,8 +512,60 @@ export class PrepareCommand {
|
|
|
232
512
|
}
|
|
233
513
|
}
|
|
234
514
|
|
|
235
|
-
//
|
|
236
|
-
if (
|
|
515
|
+
// For derived configs, try to reconcile from base config first
|
|
516
|
+
if (configDef.baseConfig && !existsSync(join(questionDir, 'meta.json'))) {
|
|
517
|
+
const reconcileResult = await reconcileQuestion({
|
|
518
|
+
targetConfig: options.memoryConfig,
|
|
519
|
+
dataset: options.dataset,
|
|
520
|
+
questionId: question.question_id,
|
|
521
|
+
preparedDataDir: options.outputDir || this.baseDir,
|
|
522
|
+
model: configDef.omModel ?? undefined,
|
|
523
|
+
});
|
|
524
|
+
|
|
525
|
+
if (reconcileResult.copied) {
|
|
526
|
+
cachedCount++;
|
|
527
|
+
completedCount++;
|
|
528
|
+
|
|
529
|
+
mainSpinner.clear();
|
|
530
|
+
console.log(
|
|
531
|
+
chalk.green(`✓`),
|
|
532
|
+
chalk.blue(`${question.question_id}`),
|
|
533
|
+
chalk.gray(`(${question.question_type})`),
|
|
534
|
+
chalk.cyan(`[reconciled from ${configDef.baseConfig}]`),
|
|
535
|
+
reconcileResult.patternsExtracted ? chalk.magenta(`[patterns]`) : '',
|
|
536
|
+
chalk.gray(`- ${completedCount}/${questionsToProcess.length}`),
|
|
537
|
+
);
|
|
538
|
+
mainSpinner.render();
|
|
539
|
+
|
|
540
|
+
updateProgress();
|
|
541
|
+
continue;
|
|
542
|
+
} else if (reconcileResult.error) {
|
|
543
|
+
// Base config data doesn't exist - need to prepare from scratch
|
|
544
|
+
// This is expected if base config hasn't been prepared yet
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
// Force regenerate: delete existing data first
|
|
549
|
+
if (options.forceRegenerate && existsSync(questionDir)) {
|
|
550
|
+
const { rm } = await import('fs/promises');
|
|
551
|
+
await rm(questionDir, { recursive: true, force: true });
|
|
552
|
+
mainSpinner.clear();
|
|
553
|
+
console.log(
|
|
554
|
+
chalk.yellow(`🗑️`),
|
|
555
|
+
chalk.blue(`${question.question_id}`),
|
|
556
|
+
chalk.gray(`- cleaned for regeneration`),
|
|
557
|
+
);
|
|
558
|
+
mainSpinner.render();
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
// Skip cache check if we're resuming from a specific message OR re-preparing from failures
|
|
562
|
+
// (when _useTempDir is set, we explicitly want to re-prepare, not use cache)
|
|
563
|
+
if (
|
|
564
|
+
!options.resumeFromMessageId &&
|
|
565
|
+
!options._useTempDir &&
|
|
566
|
+
!options.forceRegenerate &&
|
|
567
|
+
existsSync(join(questionDir, 'meta.json'))
|
|
568
|
+
) {
|
|
237
569
|
cachedCount++;
|
|
238
570
|
completedCount++;
|
|
239
571
|
|
|
@@ -265,6 +597,7 @@ export class PrepareCommand {
|
|
|
265
597
|
options,
|
|
266
598
|
model,
|
|
267
599
|
memoryOptions,
|
|
600
|
+
configDef,
|
|
268
601
|
true,
|
|
269
602
|
slotIndex,
|
|
270
603
|
activeQuestions,
|
|
@@ -275,6 +608,7 @@ export class PrepareCommand {
|
|
|
275
608
|
inProgressCount--;
|
|
276
609
|
processedCount++;
|
|
277
610
|
completedCount++;
|
|
611
|
+
successfulQuestionIds.push(question.question_id);
|
|
278
612
|
|
|
279
613
|
// Remove from active questions
|
|
280
614
|
activeQuestions.delete(slotIndex);
|
|
@@ -289,6 +623,7 @@ export class PrepareCommand {
|
|
|
289
623
|
);
|
|
290
624
|
mainSpinner.render();
|
|
291
625
|
} catch (error) {
|
|
626
|
+
console.error(`Error processing question ${question.question_id}:`, error);
|
|
292
627
|
// Check if this is a rate limit error
|
|
293
628
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
294
629
|
const isRateLimitError =
|
|
@@ -401,6 +736,50 @@ export class PrepareCommand {
|
|
|
401
736
|
);
|
|
402
737
|
}
|
|
403
738
|
|
|
739
|
+
// Atomic swap for --from-failures: move successful temp dirs to final location
|
|
740
|
+
if (options._useTempDir && successfulQuestionIds.length > 0) {
|
|
741
|
+
const { rm, rename } = await import('fs/promises');
|
|
742
|
+
const preparedDir = join(this.baseDir, options.dataset, options.memoryConfig);
|
|
743
|
+
const backupDir = join(this.baseDir, options.dataset, `.backup-${Date.now()}`);
|
|
744
|
+
|
|
745
|
+
console.log(chalk.cyan(`\n🔄 Swapping ${successfulQuestionIds.length} successfully prepared questions...`));
|
|
746
|
+
|
|
747
|
+
let swappedCount = 0;
|
|
748
|
+
let backupCount = 0;
|
|
749
|
+
|
|
750
|
+
for (const questionId of successfulQuestionIds) {
|
|
751
|
+
const tempQuestionDir = join(options._useTempDir, questionId);
|
|
752
|
+
const finalQuestionDir = join(preparedDir, questionId);
|
|
753
|
+
|
|
754
|
+
// Only proceed if temp dir exists and has meta.json (fully prepared)
|
|
755
|
+
if (existsSync(tempQuestionDir) && existsSync(join(tempQuestionDir, 'meta.json'))) {
|
|
756
|
+
// Backup existing data if it exists
|
|
757
|
+
if (existsSync(finalQuestionDir)) {
|
|
758
|
+
await mkdir(backupDir, { recursive: true });
|
|
759
|
+
const backupQuestionDir = join(backupDir, questionId);
|
|
760
|
+
await rename(finalQuestionDir, backupQuestionDir);
|
|
761
|
+
backupCount++;
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
// Move temp to final location
|
|
765
|
+
await mkdir(preparedDir, { recursive: true });
|
|
766
|
+
await rename(tempQuestionDir, finalQuestionDir);
|
|
767
|
+
swappedCount++;
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
// Clean up temp directory
|
|
772
|
+
if (existsSync(options._useTempDir)) {
|
|
773
|
+
await rm(options._useTempDir, { recursive: true, force: true });
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
console.log(chalk.green(` ✓ Swapped ${swappedCount} questions`));
|
|
777
|
+
if (backupCount > 0) {
|
|
778
|
+
console.log(chalk.gray(` Backed up ${backupCount} original directories to: ${backupDir}`));
|
|
779
|
+
console.log(chalk.gray(` You can delete the backup with: rm -rf "${backupDir}"`));
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
|
|
404
783
|
console.log(chalk.green('\n✅ Data preparation complete!\n'));
|
|
405
784
|
console.log(chalk.gray(`Prepared data saved to: ${this.baseDir}/${options.dataset}/${options.memoryConfig}/`));
|
|
406
785
|
}
|
|
@@ -410,6 +789,7 @@ export class PrepareCommand {
|
|
|
410
789
|
options: PrepareOptions,
|
|
411
790
|
model: any,
|
|
412
791
|
memoryOptions: MemoryConfigOptions,
|
|
792
|
+
configDef: import('../config').MemoryConfigDefinition,
|
|
413
793
|
isConcurrent: boolean = false,
|
|
414
794
|
slotIndex?: number,
|
|
415
795
|
activeQuestions?: Map<
|
|
@@ -419,14 +799,14 @@ export class PrepareCommand {
|
|
|
419
799
|
wmTemplates?: Record<string, any>,
|
|
420
800
|
): Promise<void> {
|
|
421
801
|
// Create fresh storage instances for this question
|
|
422
|
-
const benchmarkStore = new
|
|
802
|
+
const benchmarkStore = new PersistableInMemoryMemory();
|
|
423
803
|
const benchmarkVectorStore = new BenchmarkVectorStore();
|
|
424
804
|
|
|
425
805
|
// Initialize stores
|
|
426
|
-
await benchmarkStore.init();
|
|
806
|
+
// await benchmarkStore.init();
|
|
427
807
|
|
|
428
808
|
// Create vector index if using semantic recall
|
|
429
|
-
if (
|
|
809
|
+
if (configDef.usesSemanticRecall) {
|
|
430
810
|
await benchmarkVectorStore.createIndex({
|
|
431
811
|
indexName: 'memory_messages',
|
|
432
812
|
dimension: 1536, // text-embedding-3-small dimension
|
|
@@ -434,21 +814,16 @@ export class PrepareCommand {
|
|
|
434
814
|
});
|
|
435
815
|
}
|
|
436
816
|
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
options.memoryConfig === 'working-memory-tailored' ||
|
|
440
|
-
options.memoryConfig === 'combined' ||
|
|
441
|
-
options.memoryConfig === 'combined-tailored';
|
|
442
|
-
const usesTailoredTemplate =
|
|
443
|
-
options.memoryConfig === 'working-memory-tailored' || options.memoryConfig === 'combined-tailored';
|
|
817
|
+
// Use derived flags from consolidated config
|
|
818
|
+
const { usesWorkingMemory, usesObservationalMemory, usesShortcutOM, usesTailored } = configDef;
|
|
444
819
|
|
|
445
|
-
// Working memory must run one session (thread) at a time, in order
|
|
446
|
-
// otherwise the data will not be accurate as
|
|
447
|
-
// to build up over time, using the previous
|
|
448
|
-
if (
|
|
820
|
+
// Working memory and observational memory must run one session (thread) at a time, in order
|
|
821
|
+
// otherwise the data will not be accurate as memory is meant
|
|
822
|
+
// to build up over time, using the previous state to create the next.
|
|
823
|
+
if (configDef.requiresSequential) isConcurrent = false;
|
|
449
824
|
|
|
450
825
|
// Use custom template if available for tailored configs
|
|
451
|
-
if (
|
|
826
|
+
if (usesTailored && wmTemplates && wmTemplates[question.question_id]) {
|
|
452
827
|
memoryOptions.options.workingMemory = {
|
|
453
828
|
enabled: true,
|
|
454
829
|
template: wmTemplates[question.question_id].template,
|
|
@@ -460,26 +835,130 @@ export class PrepareCommand {
|
|
|
460
835
|
}
|
|
461
836
|
|
|
462
837
|
// Create memory with appropriate configuration
|
|
838
|
+
// Note: Using 'as any' to work around outdated BenchmarkStore types
|
|
463
839
|
const memory = new Memory({
|
|
464
|
-
storage: benchmarkStore,
|
|
465
|
-
vector:
|
|
466
|
-
|
|
467
|
-
? benchmarkVectorStore
|
|
468
|
-
: undefined,
|
|
469
|
-
embedder:
|
|
470
|
-
options.memoryConfig === 'semantic-recall' || options.memoryConfig.includes('combined')
|
|
471
|
-
? cachedOpenAI.embedding('text-embedding-3-small')
|
|
472
|
-
: undefined,
|
|
840
|
+
storage: benchmarkStore as any,
|
|
841
|
+
vector: configDef.usesSemanticRecall ? benchmarkVectorStore : undefined,
|
|
842
|
+
embedder: configDef.usesSemanticRecall ? cachedOpenAI.embedding('text-embedding-3-small') : undefined,
|
|
473
843
|
options: memoryOptions.options,
|
|
474
844
|
});
|
|
475
845
|
|
|
476
|
-
// Create
|
|
846
|
+
// Create observational memory processor if using OM config
|
|
847
|
+
let observationalMemory: ObservationalMemory | undefined;
|
|
848
|
+
let messageHistory: MessageHistory | undefined;
|
|
849
|
+
let omStorage: PersistableInMemoryMemory | undefined;
|
|
850
|
+
|
|
851
|
+
// Debug state for OM events (will be initialized after questionDir is known)
|
|
852
|
+
const omDebugState = {
|
|
853
|
+
debugLogFile: '',
|
|
854
|
+
eventCount: 0,
|
|
855
|
+
};
|
|
856
|
+
|
|
857
|
+
if (usesObservationalMemory) {
|
|
858
|
+
// Use PersistableInMemoryMemory for ObservationalMemory (has persist/hydrate)
|
|
859
|
+
omStorage = new PersistableInMemoryMemory();
|
|
860
|
+
|
|
861
|
+
// Set prompt environment variables if configured
|
|
862
|
+
if (configDef.observerUseLegacyPrompt) {
|
|
863
|
+
process.env.OM_USE_LEGACY_PROMPT = '1';
|
|
864
|
+
} else {
|
|
865
|
+
delete process.env.OM_USE_LEGACY_PROMPT;
|
|
866
|
+
}
|
|
867
|
+
if (configDef.observerUseCondensedPrompt) {
|
|
868
|
+
process.env.OM_USE_CONDENSED_PROMPT = '1';
|
|
869
|
+
} else {
|
|
870
|
+
delete process.env.OM_USE_CONDENSED_PROMPT;
|
|
871
|
+
}
|
|
872
|
+
|
|
873
|
+
// For OM: use REAL model for Observer/Reflector subagents (they need real LLMs to extract observations)
|
|
874
|
+
// For shortcut mode: use Infinity thresholds to skip observation during processing
|
|
875
|
+
// (finalize() will be called at the end to do a single observation pass)
|
|
876
|
+
// GLM shortcut uses Cerebras model (200k context), others use Gemini
|
|
877
|
+
const omModel = configDef.omModel ?? model;
|
|
878
|
+
|
|
879
|
+
observationalMemory = new ObservationalMemory({
|
|
880
|
+
obscureThreadIds: true, // can't show answer_x in context when we put the thread id in xml tags
|
|
881
|
+
storage: omStorage,
|
|
882
|
+
observation: {
|
|
883
|
+
model: omModel, // Real model for Observation
|
|
884
|
+
// Shortcut: use Infinity to skip observation during processing (finalize() does it at the end)
|
|
885
|
+
messageTokens: usesShortcutOM ? Infinity : 30000,
|
|
886
|
+
recognizePatterns: false,
|
|
887
|
+
// Allow config to override maxTokensPerBatch (default is 5000 in OM)
|
|
888
|
+
...(configDef.observerMaxTokensPerBatch && { maxTokensPerBatch: configDef.observerMaxTokensPerBatch }),
|
|
889
|
+
// Allow config to enable sequential batch processing (default is parallel)
|
|
890
|
+
},
|
|
891
|
+
reflection: {
|
|
892
|
+
model: omModel, // Real model for Reflection
|
|
893
|
+
// Shortcut: use Infinity to skip reflection during processing (finalize() does it at the end)
|
|
894
|
+
observationTokens: usesShortcutOM ? Infinity : 80000,
|
|
895
|
+
recognizePatterns: false,
|
|
896
|
+
},
|
|
897
|
+
scope: 'resource',
|
|
898
|
+
// Debug callback to log all observation events to a file
|
|
899
|
+
onDebugEvent: async (event: any) => {
|
|
900
|
+
if (!omDebugState.debugLogFile) return; // Skip if not initialized yet
|
|
901
|
+
omDebugState.eventCount++;
|
|
902
|
+
const logEntry = {
|
|
903
|
+
eventNumber: omDebugState.eventCount,
|
|
904
|
+
...event,
|
|
905
|
+
timestamp: event.timestamp.toISOString(),
|
|
906
|
+
};
|
|
907
|
+
// Write to debug log file (append)
|
|
908
|
+
await writeFile(
|
|
909
|
+
omDebugState.debugLogFile,
|
|
910
|
+
(omDebugState.eventCount === 1 ? '' : '\n') + JSON.stringify(logEntry, null, 2),
|
|
911
|
+
{ flag: 'a' },
|
|
912
|
+
);
|
|
913
|
+
// Also log summary to console
|
|
914
|
+
if (event.type === 'observation_triggered') {
|
|
915
|
+
console.log(
|
|
916
|
+
chalk.yellow(` [OM DEBUG] Observation triggered with ${event.messages?.length ?? 0} messages`),
|
|
917
|
+
);
|
|
918
|
+
} else if (event.type === 'observation_complete') {
|
|
919
|
+
console.log(chalk.green(` [OM DEBUG] Observation complete: ${event.observations?.substring(0, 100)}...`));
|
|
920
|
+
} else if (event.type === 'tokens_accumulated') {
|
|
921
|
+
console.log(
|
|
922
|
+
chalk.dim(
|
|
923
|
+
` [OM DEBUG] Tokens accumulated: ${event.sessionTokens} (total: ${event.totalPendingTokens}/${event.threshold})`,
|
|
924
|
+
),
|
|
925
|
+
);
|
|
926
|
+
}
|
|
927
|
+
},
|
|
928
|
+
});
|
|
929
|
+
|
|
930
|
+
// MessageHistory for persisting messages
|
|
931
|
+
messageHistory = new MessageHistory({
|
|
932
|
+
storage: omStorage,
|
|
933
|
+
lastMessages: 10, // Keep last 10 for context
|
|
934
|
+
});
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
// For OM: use mock model for main agent (it doesn't generate real responses during ingestion)
|
|
938
|
+
// Only the Observer/Reflector subagents need real LLMs
|
|
939
|
+
const mockAgentModel = new MockLanguageModelV2({
|
|
940
|
+
doGenerate: async () => ({
|
|
941
|
+
rawCall: { rawPrompt: null, rawSettings: {} },
|
|
942
|
+
finishReason: 'stop',
|
|
943
|
+
usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
|
|
944
|
+
content: [],
|
|
945
|
+
warnings: [],
|
|
946
|
+
}),
|
|
947
|
+
// No streaming needed for ingestion
|
|
948
|
+
});
|
|
949
|
+
|
|
950
|
+
// Create agent with appropriate model and processors
|
|
477
951
|
const agent = new Agent({
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
952
|
+
id: 'prep-agent',
|
|
953
|
+
name: 'Prep Agent',
|
|
954
|
+
instructions: usesObservationalMemory
|
|
955
|
+
? `You are a helpful assistant. Process and store conversation history.`
|
|
956
|
+
: "You are a helpful assistant. Process and store conversation history. Only store working memory information if it's in the template. Other information is not relevant",
|
|
957
|
+
model: usesObservationalMemory ? mockAgentModel : model,
|
|
958
|
+
memory: usesObservationalMemory ? undefined : memory,
|
|
959
|
+
// For OM, use processors instead of memory
|
|
960
|
+
inputProcessors: usesObservationalMemory ? [observationalMemory!] : undefined,
|
|
961
|
+
outputProcessors: usesObservationalMemory ? [messageHistory!, observationalMemory!] : undefined,
|
|
483
962
|
});
|
|
484
963
|
|
|
485
964
|
// Process all haystack sessions
|
|
@@ -503,14 +982,21 @@ export class PrepareCommand {
|
|
|
503
982
|
}
|
|
504
983
|
|
|
505
984
|
// Create output directory early to save progress
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
options.
|
|
509
|
-
options.memoryConfig,
|
|
510
|
-
question.question_id,
|
|
511
|
-
);
|
|
985
|
+
// If _useTempDir is set (from --from-failures), prepare to temp dir for atomic swap
|
|
986
|
+
const questionDir = options._useTempDir
|
|
987
|
+
? join(options._useTempDir, question.question_id)
|
|
988
|
+
: join(options.outputDir || this.baseDir, options.dataset, options.memoryConfig, question.question_id);
|
|
512
989
|
await mkdir(questionDir, { recursive: true });
|
|
513
990
|
|
|
991
|
+
// Initialize OM debug log file path now that questionDir is known
|
|
992
|
+
if (usesObservationalMemory) {
|
|
993
|
+
omDebugState.debugLogFile = join(questionDir, 'om-debug.jsonl');
|
|
994
|
+
// Clear any existing debug log
|
|
995
|
+
if (existsSync(omDebugState.debugLogFile)) {
|
|
996
|
+
await unlink(omDebugState.debugLogFile);
|
|
997
|
+
}
|
|
998
|
+
}
|
|
999
|
+
|
|
514
1000
|
// Check if this question has partial progress saved
|
|
515
1001
|
const progressFile = join(questionDir, 'progress.json');
|
|
516
1002
|
let processedSessionIds: Set<string> = new Set();
|
|
@@ -524,10 +1010,7 @@ export class PrepareCommand {
|
|
|
524
1010
|
await benchmarkStore.hydrate(dbPath);
|
|
525
1011
|
}
|
|
526
1012
|
|
|
527
|
-
if (
|
|
528
|
-
existsSync(vectorPath) &&
|
|
529
|
-
(options.memoryConfig === 'semantic-recall' || options.memoryConfig.includes('combined'))
|
|
530
|
-
) {
|
|
1013
|
+
if (existsSync(vectorPath) && configDef.usesSemanticRecall) {
|
|
531
1014
|
// console.log(chalk.gray('Loading existing vector store...'));
|
|
532
1015
|
await benchmarkVectorStore.hydrate(vectorPath);
|
|
533
1016
|
}
|
|
@@ -555,7 +1038,8 @@ export class PrepareCommand {
|
|
|
555
1038
|
}
|
|
556
1039
|
|
|
557
1040
|
// Process sessions in batches to avoid overwhelming the system
|
|
558
|
-
|
|
1041
|
+
// Working memory and observational memory must run one at a time since each session builds on memory from previous sessions
|
|
1042
|
+
const BATCH_SIZE = usesWorkingMemory || usesObservationalMemory ? 1 : 50;
|
|
559
1043
|
let processedSessions = processedSessionIds.size;
|
|
560
1044
|
|
|
561
1045
|
// Apply session offset if specified
|
|
@@ -580,14 +1064,43 @@ export class PrepareCommand {
|
|
|
580
1064
|
}
|
|
581
1065
|
|
|
582
1066
|
// Apply session limit if specified
|
|
1067
|
+
// IMPORTANT: Always include evidence sessions (answer_session_ids) to ensure the benchmark can succeed
|
|
583
1068
|
let sessionsToProcess = sessionsWithDates;
|
|
584
1069
|
if (options.sessionLimit) {
|
|
585
1070
|
const startIndex = processedSessionIds.size;
|
|
586
1071
|
const endIndex = Math.min(startIndex + options.sessionLimit, sessionsWithDates.length);
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
1072
|
+
|
|
1073
|
+
// Get evidence session IDs that contain the answer
|
|
1074
|
+
const evidenceSessionIds = new Set(question.answer_session_ids || []);
|
|
1075
|
+
|
|
1076
|
+
// Find which evidence sessions are NOT in the limited range
|
|
1077
|
+
const sessionsInRange = sessionsWithDates.slice(0, endIndex);
|
|
1078
|
+
const sessionIdsInRange = new Set(sessionsInRange.map(s => s.sessionId));
|
|
1079
|
+
|
|
1080
|
+
// Find evidence sessions that would be excluded
|
|
1081
|
+
const excludedEvidenceSessions = sessionsWithDates.filter(
|
|
1082
|
+
s => evidenceSessionIds.has(s.sessionId) && !sessionIdsInRange.has(s.sessionId),
|
|
590
1083
|
);
|
|
1084
|
+
|
|
1085
|
+
if (excludedEvidenceSessions.length > 0) {
|
|
1086
|
+
// Include the excluded evidence sessions at the end
|
|
1087
|
+
sessionsToProcess = [...sessionsInRange, ...excludedEvidenceSessions];
|
|
1088
|
+
console.log(
|
|
1089
|
+
chalk.yellow(
|
|
1090
|
+
`\n📊 Processing ${sessionsToProcess.length} sessions (${options.sessionLimit} + ${excludedEvidenceSessions.length} evidence sessions)`,
|
|
1091
|
+
),
|
|
1092
|
+
);
|
|
1093
|
+
console.log(
|
|
1094
|
+
chalk.gray(` Evidence sessions included: ${excludedEvidenceSessions.map(s => s.sessionId).join(', ')}`),
|
|
1095
|
+
);
|
|
1096
|
+
} else {
|
|
1097
|
+
sessionsToProcess = sessionsInRange;
|
|
1098
|
+
console.log(
|
|
1099
|
+
chalk.yellow(
|
|
1100
|
+
`\n📊 Processing limited to ${options.sessionLimit} sessions (${startIndex + 1} to ${endIndex})`,
|
|
1101
|
+
),
|
|
1102
|
+
);
|
|
1103
|
+
}
|
|
591
1104
|
}
|
|
592
1105
|
|
|
593
1106
|
for (let i = 0; i < sessionsToProcess.length; i += BATCH_SIZE) {
|
|
@@ -608,57 +1121,122 @@ export class PrepareCommand {
|
|
|
608
1121
|
}
|
|
609
1122
|
|
|
610
1123
|
// Process batch in parallel
|
|
611
|
-
const batchPromises = sessionBatch.map(async ({ session, sessionId }) => {
|
|
1124
|
+
const batchPromises = sessionBatch.map(async ({ session, sessionId, date }) => {
|
|
612
1125
|
// Skip if already processed
|
|
613
1126
|
if (processedSessionIds.has(sessionId)) {
|
|
614
1127
|
return;
|
|
615
1128
|
}
|
|
616
1129
|
|
|
617
|
-
//
|
|
618
|
-
const
|
|
619
|
-
|
|
1130
|
+
// Parse session date for message timestamps
|
|
1131
|
+
const sessionDate = new Date(date);
|
|
1132
|
+
|
|
1133
|
+
// Convert session to messages with historical timestamps
|
|
1134
|
+
const messages: (ModelMessage & { createdAt?: Date })[] = [];
|
|
1135
|
+
for (let turnIdx = 0; turnIdx < session.length; turnIdx++) {
|
|
1136
|
+
const turn = session[turnIdx];
|
|
620
1137
|
if (!turn.content) continue;
|
|
621
1138
|
|
|
622
1139
|
const role = turn.role === 'user' || turn.role === 'assistant' ? turn.role : 'user';
|
|
1140
|
+
// Add 5 seconds offset per message to maintain order
|
|
1141
|
+
const messageDate = new Date(sessionDate.getTime() + turnIdx * 5 * 1000);
|
|
1142
|
+
// Sanitize content that triggers Gemini's PROHIBITED_CONTENT filter
|
|
1143
|
+
const sanitizedContent = sanitizeProhibitedContent(turn.content);
|
|
623
1144
|
messages.push({
|
|
624
1145
|
role,
|
|
625
|
-
content:
|
|
1146
|
+
content: sanitizedContent,
|
|
1147
|
+
createdAt: messageDate,
|
|
626
1148
|
});
|
|
627
1149
|
}
|
|
628
1150
|
|
|
629
1151
|
if (messages.length > 0) {
|
|
630
|
-
//
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
1152
|
+
// For OM: process each message one at a time so Observer has multiple chances to make observations
|
|
1153
|
+
// If we send all messages at once, Observer only gets one chance to observe
|
|
1154
|
+
if (usesObservationalMemory) {
|
|
1155
|
+
// Process message pairs (user + assistant) one at a time
|
|
1156
|
+
for (let i = 0; i < messages.length; i += 2) {
|
|
1157
|
+
const messagePair = messages.slice(i, Math.min(i + 2, messages.length));
|
|
1158
|
+
try {
|
|
1159
|
+
await agent.generate(messagePair, {
|
|
1160
|
+
memory: {
|
|
1161
|
+
thread: sessionId,
|
|
1162
|
+
resource: resourceId,
|
|
1163
|
+
options: memoryOptions.options,
|
|
1164
|
+
},
|
|
1165
|
+
modelSettings: {
|
|
1166
|
+
temperature: 0,
|
|
1167
|
+
},
|
|
1168
|
+
});
|
|
1169
|
+
} catch (error: any) {
|
|
1170
|
+
const errorStr = error?.message || String(error);
|
|
1171
|
+
|
|
1172
|
+
// If PROHIBITED_CONTENT, dump context for debugging
|
|
1173
|
+
if (errorStr.includes('PROHIBITED_CONTENT') || errorStr.includes('blockReason')) {
|
|
1174
|
+
const dumpPath = join(questionDir, `prohibited-context-${sessionId}-${i}.json`);
|
|
1175
|
+
const contextDump = {
|
|
1176
|
+
questionId: question.question_id,
|
|
1177
|
+
sessionId,
|
|
1178
|
+
messageIndex: i,
|
|
1179
|
+
messagePair: messagePair.map(m => ({
|
|
1180
|
+
role: m.role,
|
|
1181
|
+
content: m.content,
|
|
1182
|
+
createdAt: m.createdAt?.toISOString(),
|
|
1183
|
+
})),
|
|
1184
|
+
error: errorStr,
|
|
1185
|
+
timestamp: new Date().toISOString(),
|
|
1186
|
+
};
|
|
1187
|
+
await writeFile(dumpPath, JSON.stringify(contextDump, null, 2));
|
|
1188
|
+
console.error(chalk.red(`\n⚠️ PROHIBITED_CONTENT detected! Context dumped to: ${dumpPath}`));
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1191
|
+
console.error(
|
|
1192
|
+
`Error in agent.generate for ${question.question_id}, session ${sessionId}, message ${i}:`,
|
|
1193
|
+
error,
|
|
1194
|
+
);
|
|
1195
|
+
throw error;
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
1198
|
+
} else {
|
|
1199
|
+
// For non-OM configs, process all messages at once (existing behavior)
|
|
1200
|
+
try {
|
|
1201
|
+
await agent.generate(messages, {
|
|
1202
|
+
memory: {
|
|
1203
|
+
thread: sessionId, // Use haystack session ID as thread ID
|
|
1204
|
+
resource: resourceId,
|
|
1205
|
+
options: memoryOptions.options,
|
|
1206
|
+
},
|
|
1207
|
+
modelSettings: {
|
|
1208
|
+
temperature: 0,
|
|
1209
|
+
},
|
|
1210
|
+
});
|
|
1211
|
+
} catch (error) {
|
|
1212
|
+
console.error(`Error in agent.generate for ${question.question_id}, session ${sessionId}:`, error);
|
|
1213
|
+
throw error;
|
|
1214
|
+
}
|
|
642
1215
|
}
|
|
643
1216
|
}
|
|
644
1217
|
|
|
645
1218
|
// Mark as processed
|
|
646
1219
|
processedSessionIds.add(sessionId);
|
|
647
1220
|
|
|
648
|
-
// Save progress after each session if using working memory
|
|
649
|
-
if (usesWorkingMemory) {
|
|
1221
|
+
// Save progress after each session if using working memory or observational memory
|
|
1222
|
+
if (usesWorkingMemory || usesObservationalMemory) {
|
|
650
1223
|
await writeFile(
|
|
651
1224
|
progressFile,
|
|
652
1225
|
JSON.stringify({
|
|
653
1226
|
processedSessionIds: Array.from(processedSessionIds),
|
|
654
1227
|
lastSavedDb: 'db.json',
|
|
655
1228
|
lastSavedVector: 'vector.json',
|
|
1229
|
+
lastSavedOm: usesObservationalMemory ? 'om.json' : undefined,
|
|
656
1230
|
}),
|
|
657
1231
|
);
|
|
658
1232
|
|
|
659
1233
|
// Persist current state
|
|
660
|
-
|
|
661
|
-
|
|
1234
|
+
if (usesObservationalMemory && omStorage) {
|
|
1235
|
+
await omStorage.persist(join(questionDir, 'om.json'));
|
|
1236
|
+
} else {
|
|
1237
|
+
await benchmarkStore.persist(join(questionDir, 'db.json'));
|
|
1238
|
+
}
|
|
1239
|
+
if (configDef.usesSemanticRecall) {
|
|
662
1240
|
await benchmarkVectorStore.persist(join(questionDir, 'vector.json'));
|
|
663
1241
|
}
|
|
664
1242
|
}
|
|
@@ -666,10 +1244,13 @@ export class PrepareCommand {
|
|
|
666
1244
|
|
|
667
1245
|
await Promise.all(batchPromises);
|
|
668
1246
|
|
|
669
|
-
// Fix dates for newly processed sessions
|
|
670
|
-
|
|
671
|
-
if (
|
|
672
|
-
|
|
1247
|
+
// Fix dates for newly processed sessions (only needed for non-OM configs)
|
|
1248
|
+
// OM configs pass createdAt directly on messages, so dates are correct from the start
|
|
1249
|
+
if (!usesObservationalMemory) {
|
|
1250
|
+
const newlyProcessedSessions = sessionBatch.filter(s => processedSessionIds.has(s.sessionId));
|
|
1251
|
+
if (newlyProcessedSessions.length > 0) {
|
|
1252
|
+
await this.fixSessionDates(questionDir, newlyProcessedSessions, benchmarkStore as any);
|
|
1253
|
+
}
|
|
673
1254
|
}
|
|
674
1255
|
|
|
675
1256
|
// Update processed count based on actual processed sessions
|
|
@@ -686,6 +1267,29 @@ export class PrepareCommand {
|
|
|
686
1267
|
}
|
|
687
1268
|
}
|
|
688
1269
|
|
|
1270
|
+
// Call finalize() ONLY for shortcut configs
|
|
1271
|
+
// Shortcut configs use Infinity thresholds during processing, so finalize() does all the observation work
|
|
1272
|
+
// Regular configs observe incrementally - any remaining unobserved messages appear in <other-conversation> at runtime
|
|
1273
|
+
if (usesShortcutOM && observationalMemory) {
|
|
1274
|
+
if (slotIndex !== undefined && activeQuestions) {
|
|
1275
|
+
activeQuestions.set(slotIndex, {
|
|
1276
|
+
questionId: question.question_id,
|
|
1277
|
+
status: 'Finalizing observations...',
|
|
1278
|
+
});
|
|
1279
|
+
}
|
|
1280
|
+
|
|
1281
|
+
// finalize() forces observation on all unobserved messages and triggers reflection
|
|
1282
|
+
// Use the last session's ID as the thread context
|
|
1283
|
+
const lastSessionId =
|
|
1284
|
+
sessionsWithDates[sessionsWithDates.length - 1]?.sessionId || `session_${question.question_id}`;
|
|
1285
|
+
await observationalMemory.finalize(lastSessionId, resourceId, {
|
|
1286
|
+
reflect: true,
|
|
1287
|
+
observationTokens: 20000,
|
|
1288
|
+
// For configs with token limits: use maxInputTokens to trigger mid-loop reflection
|
|
1289
|
+
maxInputTokens: configDef.omMaxInputTokens ?? undefined,
|
|
1290
|
+
});
|
|
1291
|
+
}
|
|
1292
|
+
|
|
689
1293
|
// Update status to saving
|
|
690
1294
|
if (slotIndex !== undefined && activeQuestions) {
|
|
691
1295
|
activeQuestions.set(slotIndex, {
|
|
@@ -695,10 +1299,17 @@ export class PrepareCommand {
|
|
|
695
1299
|
}
|
|
696
1300
|
|
|
697
1301
|
// Persist storage
|
|
698
|
-
|
|
1302
|
+
if (usesObservationalMemory && omStorage) {
|
|
1303
|
+
const omJsonPath = join(questionDir, 'om.json');
|
|
1304
|
+
await omStorage.persist(omJsonPath);
|
|
1305
|
+
// Make message IDs deterministic for clean git diffs
|
|
1306
|
+
await makeDeterministicIds(omJsonPath);
|
|
1307
|
+
} else {
|
|
1308
|
+
await benchmarkStore.persist(join(questionDir, 'db.json'));
|
|
1309
|
+
}
|
|
699
1310
|
|
|
700
1311
|
// Persist vector store if used
|
|
701
|
-
if (
|
|
1312
|
+
if (configDef.usesSemanticRecall) {
|
|
702
1313
|
await benchmarkVectorStore.persist(join(questionDir, 'vector.json'));
|
|
703
1314
|
}
|
|
704
1315
|
|
|
@@ -707,6 +1318,10 @@ export class PrepareCommand {
|
|
|
707
1318
|
questionId: question.question_id,
|
|
708
1319
|
questionType: question.question_type,
|
|
709
1320
|
question: question.question,
|
|
1321
|
+
improvedQuestion: question.improved_question, // Clarified version for vague/ambiguous questions
|
|
1322
|
+
improvedAnswer: question.improved_answer, // Expected answer for improved question (if different)
|
|
1323
|
+
improvementNote: question.improvement_note, // Notes about why this question failed (for tracking)
|
|
1324
|
+
requiresRetry: question.requires_retry, // If true, retry failed evaluations once (for flaky eval agent)
|
|
710
1325
|
answer: question.answer,
|
|
711
1326
|
questionDate: question.question_date,
|
|
712
1327
|
resourceId,
|
|
@@ -716,6 +1331,18 @@ export class PrepareCommand {
|
|
|
716
1331
|
sessionCount: sessionsWithDates.length,
|
|
717
1332
|
evidenceSessionIds: question.answer_session_ids,
|
|
718
1333
|
note: 'Sessions were processed in chronological order (oldest first) for working memory',
|
|
1334
|
+
// Store OM config for reproducibility
|
|
1335
|
+
...(usesObservationalMemory && {
|
|
1336
|
+
observationalMemoryConfig: {
|
|
1337
|
+
scope: 'resource',
|
|
1338
|
+
// Actual values used in ObservationalMemory constructor
|
|
1339
|
+
messageTokens: 30000,
|
|
1340
|
+
observationTokens: 40000,
|
|
1341
|
+
observationModel: configDef.omModel ?? 'google/gemini-2.5-flash',
|
|
1342
|
+
reflectionModel: configDef.omModel ?? 'google/gemini-2.5-flash',
|
|
1343
|
+
recognizePatterns: false,
|
|
1344
|
+
},
|
|
1345
|
+
}),
|
|
719
1346
|
};
|
|
720
1347
|
|
|
721
1348
|
await writeFile(join(questionDir, 'meta.json'), JSON.stringify(metadata, null, 2));
|