@mastra/longmemeval 0.0.0-error-handler-fix-20251020202607 → 0.0.0-execa-dynamic-import-20260304221256

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/CHANGELOG.md +847 -6
  2. package/LICENSE.md +15 -0
  3. package/docs/INVESTIGATION_WORKFLOW.md +314 -0
  4. package/package.json +34 -26
  5. package/scripts/download.ts +22 -1
  6. package/scripts/generate-wm-templates.ts +2 -1
  7. package/src/cli.ts +967 -155
  8. package/src/commands/__tests__/prepare.test.ts +11 -6
  9. package/src/commands/__tests__/run.test.ts +10 -8
  10. package/src/commands/clean.ts +147 -0
  11. package/src/commands/deterministic-ids.ts +126 -0
  12. package/src/commands/find-prohibited.ts +299 -0
  13. package/src/commands/investigate.ts +2437 -0
  14. package/src/commands/list-partial.ts +137 -0
  15. package/src/commands/obscure-thread-ids.ts +237 -0
  16. package/src/commands/partial-results.ts +198 -0
  17. package/src/commands/precompute-embeddings.ts +302 -0
  18. package/src/commands/prepare.ts +732 -105
  19. package/src/commands/reconcile.ts +383 -0
  20. package/src/commands/run.ts +1240 -121
  21. package/src/commands/sessions.ts +126 -0
  22. package/src/commands/sync.ts +152 -0
  23. package/src/commands/test-cleaned.ts +53 -0
  24. package/src/commands/test-incremental.ts +120 -0
  25. package/src/commands/test-messages.ts +95 -0
  26. package/src/commands/test-prohibited.ts +244 -0
  27. package/src/commands/test-thread.ts +136 -0
  28. package/src/commands/tokens.ts +625 -0
  29. package/src/config.ts +1253 -72
  30. package/src/data/types.ts +48 -6
  31. package/src/embeddings/cached-openai-embedding-model.ts +0 -1
  32. package/src/evaluation/longmemeval-metric.ts +186 -103
  33. package/src/processors/content-sanitizer.ts +153 -0
  34. package/src/processors/date-injector.ts +111 -0
  35. package/src/processors/index.ts +3 -0
  36. package/src/processors/observation-semantic-filter.ts +1142 -0
  37. package/src/retry-model.ts +5 -3
  38. package/src/storage/__tests__/benchmark-store.test.ts +18 -35
  39. package/src/storage/benchmark-store.ts +140 -153
  40. package/src/storage/benchmark-vector.ts +1 -1
  41. package/src/storage/index.ts +1 -0
  42. package/src/storage/persistable-inmemory.ts +133 -0
  43. package/src/test-utils/mock-model.ts +35 -0
@@ -0,0 +1,383 @@
1
+ /**
2
+ * Reconciliation logic for derived configs that inherit from a base config.
3
+ *
4
+ * When a config has a `baseConfig`, we need to:
5
+ * 1. Check if the target config's data exists for each question
6
+ * 2. If not, copy from the base config
7
+ * 3. Apply any enhancements (e.g., pattern extraction for om-patterns-observed)
8
+ */
9
+
10
+ import { existsSync } from 'fs';
11
+ import { readFile, writeFile, mkdir, cp } from 'fs/promises';
12
+ import { join } from 'path';
13
+ import chalk from 'chalk';
14
+ import ora from 'ora';
15
+ import { Agent } from '@mastra/core/agent';
16
+
17
+ import { getMemoryConfig } from '../config';
18
+ import type { MemoryConfigType } from '../data/types';
19
+ import { buildObserverSystemPrompt } from '@mastra/memory/processors';
20
+
21
+ // Patterns were removed from OM - this is a placeholder for legacy benchmark code
22
+ const PATTERN_INSTRUCTIONS = '';
23
+
24
+ export interface ReconcileOptions {
25
+ /** The target config we're reconciling for */
26
+ targetConfig: MemoryConfigType;
27
+ /** Dataset being used */
28
+ dataset: string;
29
+ /** Question ID to reconcile (if specific) */
30
+ questionId?: string;
31
+ /** Base directory for prepared data */
32
+ preparedDataDir: string;
33
+ /** Model to use for pattern extraction */
34
+ model?: string;
35
+ }
36
+
37
+ export interface ReconcileResult {
38
+ /** Whether reconciliation was needed */
39
+ needed: boolean;
40
+ /** Whether data was copied from base */
41
+ copied: boolean;
42
+ /** Whether patterns were extracted */
43
+ patternsExtracted: boolean;
44
+ /** Error if any */
45
+ error?: string;
46
+ }
47
+
48
+ /**
49
+ * Get the prepared data directory for a specific config and question.
50
+ */
51
+ function getQuestionDir(preparedDataDir: string, dataset: string, config: string, questionId: string): string {
52
+ return join(preparedDataDir, dataset, config, questionId);
53
+ }
54
+
55
+ /**
56
+ * Check if a question's data exists for a given config.
57
+ */
58
+ function questionDataExists(preparedDataDir: string, dataset: string, config: string, questionId: string): boolean {
59
+ const questionDir = getQuestionDir(preparedDataDir, dataset, config, questionId);
60
+ const metaPath = join(questionDir, 'meta.json');
61
+ return existsSync(metaPath);
62
+ }
63
+
64
+ /**
65
+ * Copy question data from base config to target config.
66
+ */
67
+ async function copyQuestionData(
68
+ preparedDataDir: string,
69
+ dataset: string,
70
+ baseConfig: string,
71
+ targetConfig: string,
72
+ questionId: string,
73
+ ): Promise<void> {
74
+ const sourceDir = getQuestionDir(preparedDataDir, dataset, baseConfig, questionId);
75
+ const targetDir = getQuestionDir(preparedDataDir, dataset, targetConfig, questionId);
76
+
77
+ // Ensure target directory exists
78
+ await mkdir(targetDir, { recursive: true });
79
+
80
+ // Copy all files from source to target
81
+ await cp(sourceDir, targetDir, { recursive: true });
82
+
83
+ // Update meta.json to track the base config
84
+ const metaPath = join(targetDir, 'meta.json');
85
+ if (existsSync(metaPath)) {
86
+ const meta = JSON.parse(await readFile(metaPath, 'utf-8'));
87
+ meta.baseConfig = baseConfig;
88
+ meta.reconciledAt = new Date().toISOString();
89
+ await writeFile(metaPath, JSON.stringify(meta, null, 2));
90
+ }
91
+ }
92
+
93
+ /**
94
+ * Build a prompt specifically for pattern extraction from existing observations.
95
+ */
96
+ function buildPatternExtractionPrompt(existingObservations: string): string {
97
+ return `## Existing Observations
98
+
99
+ ${existingObservations}
100
+
101
+ ---
102
+
103
+ ## Your Task
104
+
105
+ Analyze the observations above and extract patterns - recurring themes that can be grouped together.
106
+
107
+ ${PATTERN_INSTRUCTIONS}
108
+
109
+ Output ONLY the <patterns> section with your extracted patterns. Do not output <observations>, <current-task>, or <suggested-response> - those already exist.
110
+
111
+ Example output:
112
+ <patterns>
113
+ <trips>
114
+ * visited Hawaii (May 10, 2025)
115
+ * road trip to mountains (June 5, 2025)
116
+ </trips>
117
+ <purchases>
118
+ * bought new laptop (May 15, 2025)
119
+ * ordered books online (May 20, 2025)
120
+ </purchases>
121
+ </patterns>
122
+
123
+ If there are no patterns to extract (fewer than 2 related items), output:
124
+ <patterns>
125
+ </patterns>`;
126
+ }
127
+
128
+ /**
129
+ * Parse pattern-only output from the LLM.
130
+ */
131
+ function parsePatternOutput(output: string): Record<string, string[]> {
132
+ const patterns: Record<string, string[]> = {};
133
+
134
+ // Extract <patterns> content
135
+ const patternsMatch = output.match(/^[ \t]*<patterns>([\s\S]*?)^[ \t]*<\/patterns>/im);
136
+ if (!patternsMatch?.[1]) {
137
+ return patterns;
138
+ }
139
+
140
+ const patternsContent = patternsMatch[1];
141
+ // Find all named pattern tags
142
+ const patternTagRegex = /<([a-z][a-z0-9_-]*)>([\s\S]*?)<\/\1>/gi;
143
+ let patternMatch;
144
+ while ((patternMatch = patternTagRegex.exec(patternsContent)) !== null) {
145
+ const patternName = patternMatch[1];
146
+ const patternItemsRaw = patternMatch[2];
147
+ if (!patternName || !patternItemsRaw) continue;
148
+ // Extract list items (lines starting with - or *)
149
+ const items = patternItemsRaw
150
+ .split('\n')
151
+ .map(line => line.trim())
152
+ .filter(line => line.startsWith('-') || line.startsWith('*'))
153
+ .map(line => line.replace(/^[-*]\s*/, '').trim())
154
+ .filter(Boolean);
155
+ if (items.length > 0) {
156
+ patterns[patternName] = items;
157
+ }
158
+ }
159
+
160
+ return patterns;
161
+ }
162
+
163
+ /**
164
+ * Extract patterns from observations for a question.
165
+ * Uses the Observer agent to analyze existing observations and extract patterns.
166
+ */
167
+ async function extractPatternsForQuestion(
168
+ preparedDataDir: string,
169
+ dataset: string,
170
+ targetConfig: string,
171
+ questionId: string,
172
+ model: string,
173
+ ): Promise<boolean> {
174
+ const questionDir = getQuestionDir(preparedDataDir, dataset, targetConfig, questionId);
175
+ const omJsonPath = join(questionDir, 'om.json');
176
+
177
+ if (!existsSync(omJsonPath)) {
178
+ console.warn(chalk.yellow(`No om.json found for question ${questionId}`));
179
+ return false;
180
+ }
181
+
182
+ // Load the OM data
183
+ const omData = JSON.parse(await readFile(omJsonPath, 'utf-8'));
184
+
185
+ // Get the OM record (handle both old and new structure)
186
+ const records = omData.observationalMemory?.records || omData.records || [];
187
+ if (records.length === 0) {
188
+ console.warn(chalk.yellow(`No OM records found for question ${questionId}`));
189
+ return false;
190
+ }
191
+
192
+ // Get the most recent record
193
+ const record = records[records.length - 1];
194
+ const activeObservations = record.activeObservations || '';
195
+
196
+ if (!activeObservations) {
197
+ console.warn(chalk.yellow(`No active observations for question ${questionId}`));
198
+ return false;
199
+ }
200
+
201
+ // Create an agent for pattern extraction
202
+ const patternAgent = new Agent({
203
+ id: 'pattern-extractor',
204
+ name: 'pattern-extractor',
205
+ model: model as `${string}/${string}`,
206
+ instructions: buildObserverSystemPrompt(true), // Include pattern instructions
207
+ });
208
+
209
+ // Build the pattern extraction prompt
210
+ const prompt = buildPatternExtractionPrompt(activeObservations);
211
+
212
+ try {
213
+ // Call the agent to extract patterns
214
+ const result = await patternAgent.generate(prompt);
215
+ const patterns = parsePatternOutput(result.text);
216
+
217
+ // If no patterns found, that's okay - not all questions have patterns
218
+ if (Object.keys(patterns).length === 0) {
219
+ return false;
220
+ }
221
+
222
+ // Merge new patterns with any existing patterns on the record
223
+ const existingPatterns = record.patterns || {};
224
+ const mergedPatterns: Record<string, string[]> = { ...existingPatterns };
225
+
226
+ for (const [patternName, items] of Object.entries(patterns)) {
227
+ if (mergedPatterns[patternName]) {
228
+ // Merge and deduplicate
229
+ const existingItems = new Set(mergedPatterns[patternName]);
230
+ for (const item of items) {
231
+ existingItems.add(item);
232
+ }
233
+ mergedPatterns[patternName] = Array.from(existingItems);
234
+ } else {
235
+ mergedPatterns[patternName] = items;
236
+ }
237
+ }
238
+
239
+ // Update the record with patterns
240
+ record.patterns = mergedPatterns;
241
+
242
+ // Save the updated OM data
243
+ await writeFile(omJsonPath, JSON.stringify(omData, null, 2));
244
+
245
+ return true;
246
+ } catch (error) {
247
+ console.error(chalk.red(`Error extracting patterns for ${questionId}:`), error);
248
+ return false;
249
+ }
250
+ }
251
+
252
+ /**
253
+ * Reconcile a single question's data from base config to target config.
254
+ */
255
+ export async function reconcileQuestion(options: ReconcileOptions & { questionId: string }): Promise<ReconcileResult> {
256
+ const configDef = getMemoryConfig(options.targetConfig);
257
+
258
+ // If no base config, nothing to reconcile
259
+ if (!configDef.baseConfig) {
260
+ return { needed: false, copied: false, patternsExtracted: false };
261
+ }
262
+
263
+ const { preparedDataDir, dataset, targetConfig, questionId } = options;
264
+ const baseConfig = configDef.baseConfig;
265
+
266
+ // Check if target data already exists
267
+ if (questionDataExists(preparedDataDir, dataset, targetConfig, questionId)) {
268
+ return { needed: false, copied: false, patternsExtracted: false };
269
+ }
270
+
271
+ // Check if base data exists
272
+ if (!questionDataExists(preparedDataDir, dataset, baseConfig, questionId)) {
273
+ return {
274
+ needed: true,
275
+ copied: false,
276
+ patternsExtracted: false,
277
+ error: `Base config data not found for question ${questionId}`,
278
+ };
279
+ }
280
+
281
+ // Copy data from base config
282
+ await copyQuestionData(preparedDataDir, dataset, baseConfig, targetConfig, questionId);
283
+
284
+ let patternsExtracted = false;
285
+
286
+ // If this config needs pattern extraction, do it
287
+ if (configDef.recognizePatterns) {
288
+ const model = options.model || configDef.omModel || 'openai/gpt-4o';
289
+ patternsExtracted = await extractPatternsForQuestion(preparedDataDir, dataset, targetConfig, questionId, model);
290
+ }
291
+
292
+ return { needed: true, copied: true, patternsExtracted };
293
+ }
294
+
295
+ /**
296
+ * Reconcile all questions for a derived config.
297
+ * This is called before prepare/run to ensure data is available.
298
+ */
299
+ export async function reconcileFromBaseConfig(options: ReconcileOptions): Promise<{
300
+ total: number;
301
+ copied: number;
302
+ patternsExtracted: number;
303
+ errors: string[];
304
+ }> {
305
+ const configDef = getMemoryConfig(options.targetConfig);
306
+
307
+ // If no base config, nothing to reconcile
308
+ if (!configDef.baseConfig) {
309
+ return { total: 0, copied: 0, patternsExtracted: 0, errors: [] };
310
+ }
311
+
312
+ const { preparedDataDir, dataset, targetConfig, questionId } = options;
313
+ const baseConfig = configDef.baseConfig;
314
+
315
+ // If specific question, just reconcile that one
316
+ if (questionId) {
317
+ const spinner = ora(`Reconciling ${questionId} from ${baseConfig}...`).start();
318
+ const result = await reconcileQuestion({ ...options, questionId });
319
+
320
+ if (result.error) {
321
+ spinner.fail(result.error);
322
+ return { total: 1, copied: 0, patternsExtracted: 0, errors: [result.error] };
323
+ }
324
+
325
+ if (result.copied) {
326
+ spinner.succeed(
327
+ `Reconciled ${questionId} from ${baseConfig}${result.patternsExtracted ? ' (patterns extracted)' : ''}`,
328
+ );
329
+ return { total: 1, copied: 1, patternsExtracted: result.patternsExtracted ? 1 : 0, errors: [] };
330
+ }
331
+
332
+ spinner.info(`${questionId} already exists`);
333
+ return { total: 1, copied: 0, patternsExtracted: 0, errors: [] };
334
+ }
335
+
336
+ // Get all questions from base config
337
+ const baseDir = join(preparedDataDir, dataset, baseConfig);
338
+ if (!existsSync(baseDir)) {
339
+ console.log(chalk.yellow(`Base config directory not found: ${baseDir}`));
340
+ return { total: 0, copied: 0, patternsExtracted: 0, errors: [] };
341
+ }
342
+
343
+ const { readdir } = await import('fs/promises');
344
+ const questionDirs = await readdir(baseDir);
345
+ const questionIds = questionDirs.filter(d => {
346
+ const metaPath = join(baseDir, d, 'meta.json');
347
+ return existsSync(metaPath);
348
+ });
349
+
350
+ if (questionIds.length === 0) {
351
+ console.log(chalk.yellow(`No prepared questions found in base config: ${baseConfig}`));
352
+ return { total: 0, copied: 0, patternsExtracted: 0, errors: [] };
353
+ }
354
+
355
+ console.log(
356
+ chalk.blue(`\nšŸ“¦ Reconciling ${questionIds.length} questions from ${baseConfig} to ${targetConfig}...\n`),
357
+ );
358
+
359
+ let copied = 0;
360
+ let patternsExtracted = 0;
361
+ const errors: string[] = [];
362
+
363
+ for (const qId of questionIds) {
364
+ const result = await reconcileQuestion({ ...options, questionId: qId });
365
+
366
+ if (result.error) {
367
+ errors.push(result.error);
368
+ } else if (result.copied) {
369
+ copied++;
370
+ if (result.patternsExtracted) {
371
+ patternsExtracted++;
372
+ }
373
+ }
374
+ }
375
+
376
+ console.log(
377
+ chalk.green(
378
+ `\nāœ“ Reconciliation complete: ${copied} copied, ${patternsExtracted} patterns extracted, ${errors.length} errors\n`,
379
+ ),
380
+ );
381
+
382
+ return { total: questionIds.length, copied, patternsExtracted, errors };
383
+ }