@mastra/longmemeval 0.0.0-error-handler-fix-20251020202607 ā 0.0.0-execa-dynamic-import-20260304221256
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +847 -6
- package/LICENSE.md +15 -0
- package/docs/INVESTIGATION_WORKFLOW.md +314 -0
- package/package.json +34 -26
- package/scripts/download.ts +22 -1
- package/scripts/generate-wm-templates.ts +2 -1
- package/src/cli.ts +967 -155
- package/src/commands/__tests__/prepare.test.ts +11 -6
- package/src/commands/__tests__/run.test.ts +10 -8
- package/src/commands/clean.ts +147 -0
- package/src/commands/deterministic-ids.ts +126 -0
- package/src/commands/find-prohibited.ts +299 -0
- package/src/commands/investigate.ts +2437 -0
- package/src/commands/list-partial.ts +137 -0
- package/src/commands/obscure-thread-ids.ts +237 -0
- package/src/commands/partial-results.ts +198 -0
- package/src/commands/precompute-embeddings.ts +302 -0
- package/src/commands/prepare.ts +732 -105
- package/src/commands/reconcile.ts +383 -0
- package/src/commands/run.ts +1240 -121
- package/src/commands/sessions.ts +126 -0
- package/src/commands/sync.ts +152 -0
- package/src/commands/test-cleaned.ts +53 -0
- package/src/commands/test-incremental.ts +120 -0
- package/src/commands/test-messages.ts +95 -0
- package/src/commands/test-prohibited.ts +244 -0
- package/src/commands/test-thread.ts +136 -0
- package/src/commands/tokens.ts +625 -0
- package/src/config.ts +1253 -72
- package/src/data/types.ts +48 -6
- package/src/embeddings/cached-openai-embedding-model.ts +0 -1
- package/src/evaluation/longmemeval-metric.ts +186 -103
- package/src/processors/content-sanitizer.ts +153 -0
- package/src/processors/date-injector.ts +111 -0
- package/src/processors/index.ts +3 -0
- package/src/processors/observation-semantic-filter.ts +1142 -0
- package/src/retry-model.ts +5 -3
- package/src/storage/__tests__/benchmark-store.test.ts +18 -35
- package/src/storage/benchmark-store.ts +140 -153
- package/src/storage/benchmark-vector.ts +1 -1
- package/src/storage/index.ts +1 -0
- package/src/storage/persistable-inmemory.ts +133 -0
- package/src/test-utils/mock-model.ts +35 -0
|
@@ -0,0 +1,383 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reconciliation logic for derived configs that inherit from a base config.
|
|
3
|
+
*
|
|
4
|
+
* When a config has a `baseConfig`, we need to:
|
|
5
|
+
* 1. Check if the target config's data exists for each question
|
|
6
|
+
* 2. If not, copy from the base config
|
|
7
|
+
* 3. Apply any enhancements (e.g., pattern extraction for om-patterns-observed)
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { existsSync } from 'fs';
|
|
11
|
+
import { readFile, writeFile, mkdir, cp } from 'fs/promises';
|
|
12
|
+
import { join } from 'path';
|
|
13
|
+
import chalk from 'chalk';
|
|
14
|
+
import ora from 'ora';
|
|
15
|
+
import { Agent } from '@mastra/core/agent';
|
|
16
|
+
|
|
17
|
+
import { getMemoryConfig } from '../config';
|
|
18
|
+
import type { MemoryConfigType } from '../data/types';
|
|
19
|
+
import { buildObserverSystemPrompt } from '@mastra/memory/processors';
|
|
20
|
+
|
|
21
|
+
// Patterns were removed from OM - this is a placeholder for legacy benchmark code
|
|
22
|
+
const PATTERN_INSTRUCTIONS = '';
|
|
23
|
+
|
|
24
|
+
export interface ReconcileOptions {
|
|
25
|
+
/** The target config we're reconciling for */
|
|
26
|
+
targetConfig: MemoryConfigType;
|
|
27
|
+
/** Dataset being used */
|
|
28
|
+
dataset: string;
|
|
29
|
+
/** Question ID to reconcile (if specific) */
|
|
30
|
+
questionId?: string;
|
|
31
|
+
/** Base directory for prepared data */
|
|
32
|
+
preparedDataDir: string;
|
|
33
|
+
/** Model to use for pattern extraction */
|
|
34
|
+
model?: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface ReconcileResult {
|
|
38
|
+
/** Whether reconciliation was needed */
|
|
39
|
+
needed: boolean;
|
|
40
|
+
/** Whether data was copied from base */
|
|
41
|
+
copied: boolean;
|
|
42
|
+
/** Whether patterns were extracted */
|
|
43
|
+
patternsExtracted: boolean;
|
|
44
|
+
/** Error if any */
|
|
45
|
+
error?: string;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Get the prepared data directory for a specific config and question.
|
|
50
|
+
*/
|
|
51
|
+
function getQuestionDir(preparedDataDir: string, dataset: string, config: string, questionId: string): string {
|
|
52
|
+
return join(preparedDataDir, dataset, config, questionId);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Check if a question's data exists for a given config.
|
|
57
|
+
*/
|
|
58
|
+
function questionDataExists(preparedDataDir: string, dataset: string, config: string, questionId: string): boolean {
|
|
59
|
+
const questionDir = getQuestionDir(preparedDataDir, dataset, config, questionId);
|
|
60
|
+
const metaPath = join(questionDir, 'meta.json');
|
|
61
|
+
return existsSync(metaPath);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Copy question data from base config to target config.
|
|
66
|
+
*/
|
|
67
|
+
async function copyQuestionData(
|
|
68
|
+
preparedDataDir: string,
|
|
69
|
+
dataset: string,
|
|
70
|
+
baseConfig: string,
|
|
71
|
+
targetConfig: string,
|
|
72
|
+
questionId: string,
|
|
73
|
+
): Promise<void> {
|
|
74
|
+
const sourceDir = getQuestionDir(preparedDataDir, dataset, baseConfig, questionId);
|
|
75
|
+
const targetDir = getQuestionDir(preparedDataDir, dataset, targetConfig, questionId);
|
|
76
|
+
|
|
77
|
+
// Ensure target directory exists
|
|
78
|
+
await mkdir(targetDir, { recursive: true });
|
|
79
|
+
|
|
80
|
+
// Copy all files from source to target
|
|
81
|
+
await cp(sourceDir, targetDir, { recursive: true });
|
|
82
|
+
|
|
83
|
+
// Update meta.json to track the base config
|
|
84
|
+
const metaPath = join(targetDir, 'meta.json');
|
|
85
|
+
if (existsSync(metaPath)) {
|
|
86
|
+
const meta = JSON.parse(await readFile(metaPath, 'utf-8'));
|
|
87
|
+
meta.baseConfig = baseConfig;
|
|
88
|
+
meta.reconciledAt = new Date().toISOString();
|
|
89
|
+
await writeFile(metaPath, JSON.stringify(meta, null, 2));
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Build a prompt specifically for pattern extraction from existing observations.
|
|
95
|
+
*/
|
|
96
|
+
function buildPatternExtractionPrompt(existingObservations: string): string {
|
|
97
|
+
return `## Existing Observations
|
|
98
|
+
|
|
99
|
+
${existingObservations}
|
|
100
|
+
|
|
101
|
+
---
|
|
102
|
+
|
|
103
|
+
## Your Task
|
|
104
|
+
|
|
105
|
+
Analyze the observations above and extract patterns - recurring themes that can be grouped together.
|
|
106
|
+
|
|
107
|
+
${PATTERN_INSTRUCTIONS}
|
|
108
|
+
|
|
109
|
+
Output ONLY the <patterns> section with your extracted patterns. Do not output <observations>, <current-task>, or <suggested-response> - those already exist.
|
|
110
|
+
|
|
111
|
+
Example output:
|
|
112
|
+
<patterns>
|
|
113
|
+
<trips>
|
|
114
|
+
* visited Hawaii (May 10, 2025)
|
|
115
|
+
* road trip to mountains (June 5, 2025)
|
|
116
|
+
</trips>
|
|
117
|
+
<purchases>
|
|
118
|
+
* bought new laptop (May 15, 2025)
|
|
119
|
+
* ordered books online (May 20, 2025)
|
|
120
|
+
</purchases>
|
|
121
|
+
</patterns>
|
|
122
|
+
|
|
123
|
+
If there are no patterns to extract (fewer than 2 related items), output:
|
|
124
|
+
<patterns>
|
|
125
|
+
</patterns>`;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Parse pattern-only output from the LLM.
|
|
130
|
+
*/
|
|
131
|
+
function parsePatternOutput(output: string): Record<string, string[]> {
|
|
132
|
+
const patterns: Record<string, string[]> = {};
|
|
133
|
+
|
|
134
|
+
// Extract <patterns> content
|
|
135
|
+
const patternsMatch = output.match(/^[ \t]*<patterns>([\s\S]*?)^[ \t]*<\/patterns>/im);
|
|
136
|
+
if (!patternsMatch?.[1]) {
|
|
137
|
+
return patterns;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const patternsContent = patternsMatch[1];
|
|
141
|
+
// Find all named pattern tags
|
|
142
|
+
const patternTagRegex = /<([a-z][a-z0-9_-]*)>([\s\S]*?)<\/\1>/gi;
|
|
143
|
+
let patternMatch;
|
|
144
|
+
while ((patternMatch = patternTagRegex.exec(patternsContent)) !== null) {
|
|
145
|
+
const patternName = patternMatch[1];
|
|
146
|
+
const patternItemsRaw = patternMatch[2];
|
|
147
|
+
if (!patternName || !patternItemsRaw) continue;
|
|
148
|
+
// Extract list items (lines starting with - or *)
|
|
149
|
+
const items = patternItemsRaw
|
|
150
|
+
.split('\n')
|
|
151
|
+
.map(line => line.trim())
|
|
152
|
+
.filter(line => line.startsWith('-') || line.startsWith('*'))
|
|
153
|
+
.map(line => line.replace(/^[-*]\s*/, '').trim())
|
|
154
|
+
.filter(Boolean);
|
|
155
|
+
if (items.length > 0) {
|
|
156
|
+
patterns[patternName] = items;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
return patterns;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Extract patterns from observations for a question.
|
|
165
|
+
* Uses the Observer agent to analyze existing observations and extract patterns.
|
|
166
|
+
*/
|
|
167
|
+
async function extractPatternsForQuestion(
|
|
168
|
+
preparedDataDir: string,
|
|
169
|
+
dataset: string,
|
|
170
|
+
targetConfig: string,
|
|
171
|
+
questionId: string,
|
|
172
|
+
model: string,
|
|
173
|
+
): Promise<boolean> {
|
|
174
|
+
const questionDir = getQuestionDir(preparedDataDir, dataset, targetConfig, questionId);
|
|
175
|
+
const omJsonPath = join(questionDir, 'om.json');
|
|
176
|
+
|
|
177
|
+
if (!existsSync(omJsonPath)) {
|
|
178
|
+
console.warn(chalk.yellow(`No om.json found for question ${questionId}`));
|
|
179
|
+
return false;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// Load the OM data
|
|
183
|
+
const omData = JSON.parse(await readFile(omJsonPath, 'utf-8'));
|
|
184
|
+
|
|
185
|
+
// Get the OM record (handle both old and new structure)
|
|
186
|
+
const records = omData.observationalMemory?.records || omData.records || [];
|
|
187
|
+
if (records.length === 0) {
|
|
188
|
+
console.warn(chalk.yellow(`No OM records found for question ${questionId}`));
|
|
189
|
+
return false;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// Get the most recent record
|
|
193
|
+
const record = records[records.length - 1];
|
|
194
|
+
const activeObservations = record.activeObservations || '';
|
|
195
|
+
|
|
196
|
+
if (!activeObservations) {
|
|
197
|
+
console.warn(chalk.yellow(`No active observations for question ${questionId}`));
|
|
198
|
+
return false;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// Create an agent for pattern extraction
|
|
202
|
+
const patternAgent = new Agent({
|
|
203
|
+
id: 'pattern-extractor',
|
|
204
|
+
name: 'pattern-extractor',
|
|
205
|
+
model: model as `${string}/${string}`,
|
|
206
|
+
instructions: buildObserverSystemPrompt(true), // Include pattern instructions
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
// Build the pattern extraction prompt
|
|
210
|
+
const prompt = buildPatternExtractionPrompt(activeObservations);
|
|
211
|
+
|
|
212
|
+
try {
|
|
213
|
+
// Call the agent to extract patterns
|
|
214
|
+
const result = await patternAgent.generate(prompt);
|
|
215
|
+
const patterns = parsePatternOutput(result.text);
|
|
216
|
+
|
|
217
|
+
// If no patterns found, that's okay - not all questions have patterns
|
|
218
|
+
if (Object.keys(patterns).length === 0) {
|
|
219
|
+
return false;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
// Merge new patterns with any existing patterns on the record
|
|
223
|
+
const existingPatterns = record.patterns || {};
|
|
224
|
+
const mergedPatterns: Record<string, string[]> = { ...existingPatterns };
|
|
225
|
+
|
|
226
|
+
for (const [patternName, items] of Object.entries(patterns)) {
|
|
227
|
+
if (mergedPatterns[patternName]) {
|
|
228
|
+
// Merge and deduplicate
|
|
229
|
+
const existingItems = new Set(mergedPatterns[patternName]);
|
|
230
|
+
for (const item of items) {
|
|
231
|
+
existingItems.add(item);
|
|
232
|
+
}
|
|
233
|
+
mergedPatterns[patternName] = Array.from(existingItems);
|
|
234
|
+
} else {
|
|
235
|
+
mergedPatterns[patternName] = items;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
// Update the record with patterns
|
|
240
|
+
record.patterns = mergedPatterns;
|
|
241
|
+
|
|
242
|
+
// Save the updated OM data
|
|
243
|
+
await writeFile(omJsonPath, JSON.stringify(omData, null, 2));
|
|
244
|
+
|
|
245
|
+
return true;
|
|
246
|
+
} catch (error) {
|
|
247
|
+
console.error(chalk.red(`Error extracting patterns for ${questionId}:`), error);
|
|
248
|
+
return false;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* Reconcile a single question's data from base config to target config.
|
|
254
|
+
*/
|
|
255
|
+
export async function reconcileQuestion(options: ReconcileOptions & { questionId: string }): Promise<ReconcileResult> {
|
|
256
|
+
const configDef = getMemoryConfig(options.targetConfig);
|
|
257
|
+
|
|
258
|
+
// If no base config, nothing to reconcile
|
|
259
|
+
if (!configDef.baseConfig) {
|
|
260
|
+
return { needed: false, copied: false, patternsExtracted: false };
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
const { preparedDataDir, dataset, targetConfig, questionId } = options;
|
|
264
|
+
const baseConfig = configDef.baseConfig;
|
|
265
|
+
|
|
266
|
+
// Check if target data already exists
|
|
267
|
+
if (questionDataExists(preparedDataDir, dataset, targetConfig, questionId)) {
|
|
268
|
+
return { needed: false, copied: false, patternsExtracted: false };
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// Check if base data exists
|
|
272
|
+
if (!questionDataExists(preparedDataDir, dataset, baseConfig, questionId)) {
|
|
273
|
+
return {
|
|
274
|
+
needed: true,
|
|
275
|
+
copied: false,
|
|
276
|
+
patternsExtracted: false,
|
|
277
|
+
error: `Base config data not found for question ${questionId}`,
|
|
278
|
+
};
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// Copy data from base config
|
|
282
|
+
await copyQuestionData(preparedDataDir, dataset, baseConfig, targetConfig, questionId);
|
|
283
|
+
|
|
284
|
+
let patternsExtracted = false;
|
|
285
|
+
|
|
286
|
+
// If this config needs pattern extraction, do it
|
|
287
|
+
if (configDef.recognizePatterns) {
|
|
288
|
+
const model = options.model || configDef.omModel || 'openai/gpt-4o';
|
|
289
|
+
patternsExtracted = await extractPatternsForQuestion(preparedDataDir, dataset, targetConfig, questionId, model);
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
return { needed: true, copied: true, patternsExtracted };
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* Reconcile all questions for a derived config.
|
|
297
|
+
* This is called before prepare/run to ensure data is available.
|
|
298
|
+
*/
|
|
299
|
+
export async function reconcileFromBaseConfig(options: ReconcileOptions): Promise<{
|
|
300
|
+
total: number;
|
|
301
|
+
copied: number;
|
|
302
|
+
patternsExtracted: number;
|
|
303
|
+
errors: string[];
|
|
304
|
+
}> {
|
|
305
|
+
const configDef = getMemoryConfig(options.targetConfig);
|
|
306
|
+
|
|
307
|
+
// If no base config, nothing to reconcile
|
|
308
|
+
if (!configDef.baseConfig) {
|
|
309
|
+
return { total: 0, copied: 0, patternsExtracted: 0, errors: [] };
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
const { preparedDataDir, dataset, targetConfig, questionId } = options;
|
|
313
|
+
const baseConfig = configDef.baseConfig;
|
|
314
|
+
|
|
315
|
+
// If specific question, just reconcile that one
|
|
316
|
+
if (questionId) {
|
|
317
|
+
const spinner = ora(`Reconciling ${questionId} from ${baseConfig}...`).start();
|
|
318
|
+
const result = await reconcileQuestion({ ...options, questionId });
|
|
319
|
+
|
|
320
|
+
if (result.error) {
|
|
321
|
+
spinner.fail(result.error);
|
|
322
|
+
return { total: 1, copied: 0, patternsExtracted: 0, errors: [result.error] };
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
if (result.copied) {
|
|
326
|
+
spinner.succeed(
|
|
327
|
+
`Reconciled ${questionId} from ${baseConfig}${result.patternsExtracted ? ' (patterns extracted)' : ''}`,
|
|
328
|
+
);
|
|
329
|
+
return { total: 1, copied: 1, patternsExtracted: result.patternsExtracted ? 1 : 0, errors: [] };
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
spinner.info(`${questionId} already exists`);
|
|
333
|
+
return { total: 1, copied: 0, patternsExtracted: 0, errors: [] };
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
// Get all questions from base config
|
|
337
|
+
const baseDir = join(preparedDataDir, dataset, baseConfig);
|
|
338
|
+
if (!existsSync(baseDir)) {
|
|
339
|
+
console.log(chalk.yellow(`Base config directory not found: ${baseDir}`));
|
|
340
|
+
return { total: 0, copied: 0, patternsExtracted: 0, errors: [] };
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
const { readdir } = await import('fs/promises');
|
|
344
|
+
const questionDirs = await readdir(baseDir);
|
|
345
|
+
const questionIds = questionDirs.filter(d => {
|
|
346
|
+
const metaPath = join(baseDir, d, 'meta.json');
|
|
347
|
+
return existsSync(metaPath);
|
|
348
|
+
});
|
|
349
|
+
|
|
350
|
+
if (questionIds.length === 0) {
|
|
351
|
+
console.log(chalk.yellow(`No prepared questions found in base config: ${baseConfig}`));
|
|
352
|
+
return { total: 0, copied: 0, patternsExtracted: 0, errors: [] };
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
console.log(
|
|
356
|
+
chalk.blue(`\nš¦ Reconciling ${questionIds.length} questions from ${baseConfig} to ${targetConfig}...\n`),
|
|
357
|
+
);
|
|
358
|
+
|
|
359
|
+
let copied = 0;
|
|
360
|
+
let patternsExtracted = 0;
|
|
361
|
+
const errors: string[] = [];
|
|
362
|
+
|
|
363
|
+
for (const qId of questionIds) {
|
|
364
|
+
const result = await reconcileQuestion({ ...options, questionId: qId });
|
|
365
|
+
|
|
366
|
+
if (result.error) {
|
|
367
|
+
errors.push(result.error);
|
|
368
|
+
} else if (result.copied) {
|
|
369
|
+
copied++;
|
|
370
|
+
if (result.patternsExtracted) {
|
|
371
|
+
patternsExtracted++;
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
console.log(
|
|
377
|
+
chalk.green(
|
|
378
|
+
`\nā Reconciliation complete: ${copied} copied, ${patternsExtracted} patterns extracted, ${errors.length} errors\n`,
|
|
379
|
+
),
|
|
380
|
+
);
|
|
381
|
+
|
|
382
|
+
return { total: questionIds.length, copied, patternsExtracted, errors };
|
|
383
|
+
}
|