@mastra/longmemeval 0.0.0-error-handler-fix-20251020202607 → 0.0.0-execa-dynamic-import-20260304221256

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/CHANGELOG.md +847 -6
  2. package/LICENSE.md +15 -0
  3. package/docs/INVESTIGATION_WORKFLOW.md +314 -0
  4. package/package.json +34 -26
  5. package/scripts/download.ts +22 -1
  6. package/scripts/generate-wm-templates.ts +2 -1
  7. package/src/cli.ts +967 -155
  8. package/src/commands/__tests__/prepare.test.ts +11 -6
  9. package/src/commands/__tests__/run.test.ts +10 -8
  10. package/src/commands/clean.ts +147 -0
  11. package/src/commands/deterministic-ids.ts +126 -0
  12. package/src/commands/find-prohibited.ts +299 -0
  13. package/src/commands/investigate.ts +2437 -0
  14. package/src/commands/list-partial.ts +137 -0
  15. package/src/commands/obscure-thread-ids.ts +237 -0
  16. package/src/commands/partial-results.ts +198 -0
  17. package/src/commands/precompute-embeddings.ts +302 -0
  18. package/src/commands/prepare.ts +732 -105
  19. package/src/commands/reconcile.ts +383 -0
  20. package/src/commands/run.ts +1240 -121
  21. package/src/commands/sessions.ts +126 -0
  22. package/src/commands/sync.ts +152 -0
  23. package/src/commands/test-cleaned.ts +53 -0
  24. package/src/commands/test-incremental.ts +120 -0
  25. package/src/commands/test-messages.ts +95 -0
  26. package/src/commands/test-prohibited.ts +244 -0
  27. package/src/commands/test-thread.ts +136 -0
  28. package/src/commands/tokens.ts +625 -0
  29. package/src/config.ts +1253 -72
  30. package/src/data/types.ts +48 -6
  31. package/src/embeddings/cached-openai-embedding-model.ts +0 -1
  32. package/src/evaluation/longmemeval-metric.ts +186 -103
  33. package/src/processors/content-sanitizer.ts +153 -0
  34. package/src/processors/date-injector.ts +111 -0
  35. package/src/processors/index.ts +3 -0
  36. package/src/processors/observation-semantic-filter.ts +1142 -0
  37. package/src/retry-model.ts +5 -3
  38. package/src/storage/__tests__/benchmark-store.test.ts +18 -35
  39. package/src/storage/benchmark-store.ts +140 -153
  40. package/src/storage/benchmark-vector.ts +1 -1
  41. package/src/storage/index.ts +1 -0
  42. package/src/storage/persistable-inmemory.ts +133 -0
  43. package/src/test-utils/mock-model.ts +35 -0
@@ -0,0 +1,302 @@
1
+ /**
2
+ * Precompute embeddings for all prepared questions
3
+ *
4
+ * This command indexes all observations from prepared data and caches
5
+ * the embeddings to disk. This makes subsequent benchmark runs much faster
6
+ * since embeddings don't need to be computed at runtime.
7
+ *
8
+ * Usage:
9
+ * pnpm precompute-embeddings -d longmemeval_s -c observational-memory
10
+ * pnpm precompute-embeddings -d longmemeval_s -c observational-memory --offset 100 --subset 50
11
+ */
12
+
13
+ import { existsSync, readdirSync, readFileSync, mkdirSync } from 'fs';
14
+ import { join } from 'path';
15
+ import ora from 'ora';
16
+ import chalk from 'chalk';
17
+ import xxhash from 'xxhash-wasm';
18
+ import { embedMany } from 'ai';
19
+ import { fastembed } from '@mastra/fastembed';
20
+ import { writeFileSync } from 'fs';
21
+
22
+ import type { DatasetType, MemoryConfigType } from '../data/types';
23
+ import { getMemoryConfig } from '../config';
24
+
25
+ export interface PrecomputeEmbeddingsOptions {
26
+ dataset: DatasetType;
27
+ memoryConfig: MemoryConfigType;
28
+ preparedDataDir?: string;
29
+ offset?: number;
30
+ subset?: number;
31
+ batchSize?: number;
32
+ cooldown?: number; // Cooldown in ms between questions
33
+ }
34
+
35
+ interface ObservationalMemoryRecord {
36
+ activeObservations?: string;
37
+ }
38
+
39
+ interface ObservationalMemoryData {
40
+ // New format: observationalMemory is an array of [key, [record, ...]] tuples
41
+ // The second element is itself an array containing the record object(s)
42
+ observationalMemory?: Array<[string, ObservationalMemoryRecord[]]>;
43
+ // Old format: direct record object
44
+ record?: ObservationalMemoryRecord;
45
+ }
46
+
47
+ export class PrecomputeEmbeddingsCommand {
48
+ private hasher = xxhash();
49
+
50
+ async run(options: PrecomputeEmbeddingsOptions): Promise<void> {
51
+ const spinner = ora('Loading prepared data...').start();
52
+
53
+ const configDef = getMemoryConfig(options.memoryConfig);
54
+
55
+ // Determine which config to read from (handle readOnlyConfig)
56
+ const effectiveConfig =
57
+ configDef.readOnlyConfig && configDef.baseConfig ? configDef.baseConfig : options.memoryConfig;
58
+
59
+ const baseDir = options.preparedDataDir || join(process.cwd(), 'prepared-data');
60
+ const preparedDir = join(baseDir, options.dataset, effectiveConfig);
61
+
62
+ if (!existsSync(preparedDir)) {
63
+ spinner.fail(`Prepared data directory not found: ${preparedDir}`);
64
+ return;
65
+ }
66
+
67
+ // Find all question directories
68
+ const questionDirs = readdirSync(preparedDir, { withFileTypes: true })
69
+ .filter(d => d.isDirectory() && !d.name.startsWith('.'))
70
+ .map(d => d.name);
71
+
72
+ if (questionDirs.length === 0) {
73
+ spinner.fail('No prepared questions found');
74
+ return;
75
+ }
76
+
77
+ // Apply offset and subset
78
+ let selectedQuestions = questionDirs;
79
+ if (options.offset) {
80
+ selectedQuestions = selectedQuestions.slice(options.offset);
81
+ }
82
+ if (options.subset) {
83
+ selectedQuestions = selectedQuestions.slice(0, options.subset);
84
+ }
85
+
86
+ spinner.succeed(`Found ${questionDirs.length} questions, processing ${selectedQuestions.length}`);
87
+
88
+ // Create shared cache directory with model subdirectory
89
+ const modelName = 'fastembed-small';
90
+ const cacheDir = join(preparedDir, '..', '.embeddings-cache', modelName);
91
+ if (!existsSync(cacheDir)) {
92
+ mkdirSync(cacheDir, { recursive: true });
93
+ }
94
+
95
+ console.log(chalk.gray(`Cache directory: ${cacheDir}`));
96
+ console.log(chalk.gray(`Embedding model: ${modelName}`));
97
+
98
+ // Track stats
99
+ let totalObservations = 0;
100
+ let totalQuestions = 0;
101
+ let cachedCount = 0;
102
+ let newCount = 0;
103
+ let questionsCached = 0;
104
+ let questionsNew = 0;
105
+ let errorCount = 0;
106
+
107
+ const hasher = await this.hasher;
108
+ const batchSize = options.batchSize || 25; // Small batches to reduce CPU spikes
109
+ const cooldown = options.cooldown ?? 1000; // Default 1 second between questions
110
+
111
+ // Process each question
112
+ for (let i = 0; i < selectedQuestions.length; i++) {
113
+ const questionId = selectedQuestions[i];
114
+ const questionDir = join(preparedDir, questionId);
115
+ const omJsonPath = join(questionDir, 'om.json');
116
+
117
+ const progress = `[${i + 1}/${selectedQuestions.length}]`;
118
+ spinner.start(`${progress} Processing ${questionId}...`);
119
+
120
+ if (!existsSync(omJsonPath)) {
121
+ spinner.warn(`${progress} ${questionId}: No om.json found, skipping`);
122
+ continue;
123
+ }
124
+
125
+ try {
126
+ // Load observations from om.json
127
+ const omData: ObservationalMemoryData = JSON.parse(readFileSync(omJsonPath, 'utf-8'));
128
+
129
+ // Extract observations - handle both new format (array of tuples) and old format
130
+ let observations = '';
131
+ if (omData.observationalMemory && Array.isArray(omData.observationalMemory)) {
132
+ // New format: [[key, [record, ...]], ...] - get activeObservations from first record in first tuple
133
+ // Structure: observationalMemory[0] = [id, [record_object, ...]]
134
+ const recordArray = omData.observationalMemory[0]?.[1];
135
+ const firstRecord = Array.isArray(recordArray) ? recordArray[0] : undefined;
136
+ observations = firstRecord?.activeObservations || '';
137
+ } else if (omData.record?.activeObservations) {
138
+ // Old format: direct record object
139
+ observations = omData.record.activeObservations;
140
+ }
141
+
142
+ if (!observations) {
143
+ spinner.info(`${progress} ${questionId}: No observations found`);
144
+ continue;
145
+ }
146
+
147
+ // Parse observation lines
148
+ const lines = this.parseObservationLines(observations);
149
+ totalObservations += lines.length;
150
+
151
+ // Check which lines need embedding
152
+ const uncachedLines: { line: string; hash: string }[] = [];
153
+
154
+ for (const line of lines) {
155
+ const hash = hasher.h32ToString(line);
156
+ const cachePath = join(cacheDir, `${hash}.json`);
157
+
158
+ if (existsSync(cachePath)) {
159
+ cachedCount++;
160
+ } else {
161
+ uncachedLines.push({ line, hash });
162
+ }
163
+ }
164
+
165
+ if (uncachedLines.length === 0) {
166
+ spinner.succeed(`${progress} ${questionId}: ${lines.length} observations (all cached)`);
167
+ continue;
168
+ }
169
+
170
+ // Embed in batches with micro-cooldowns to prevent CPU overload
171
+ const batchCooldown = Math.min(cooldown / 4, 250); // 250ms max between batches
172
+ const totalBatches = Math.ceil(uncachedLines.length / batchSize);
173
+
174
+ for (let b = 0; b < uncachedLines.length; b += batchSize) {
175
+ const batchNum = Math.floor(b / batchSize) + 1;
176
+ const batch = uncachedLines.slice(b, b + batchSize);
177
+
178
+ spinner.text = `${progress} ${questionId}: embedding batch ${batchNum}/${totalBatches} (${batch.length} items)`;
179
+
180
+ const { embeddings } = await embedMany({
181
+ model: fastembed.small,
182
+ values: batch.map(l => l.line),
183
+ });
184
+
185
+ // Save to cache
186
+ for (let j = 0; j < batch.length; j++) {
187
+ const { hash } = batch[j];
188
+ const embedding = embeddings[j];
189
+ const cachePath = join(cacheDir, `${hash}.json`);
190
+
191
+ try {
192
+ writeFileSync(cachePath, JSON.stringify(embedding));
193
+ newCount++;
194
+ } catch {
195
+ errorCount++;
196
+ }
197
+ }
198
+
199
+ // Micro-cooldown between batches within a question
200
+ if (batchCooldown > 0 && b + batchSize < uncachedLines.length) {
201
+ await new Promise(resolve => setTimeout(resolve, batchCooldown));
202
+ }
203
+ }
204
+
205
+ // Also embed the question(s) from meta.json
206
+ const metaJsonPath = join(questionDir, 'meta.json');
207
+ let questionStats = '';
208
+ if (existsSync(metaJsonPath)) {
209
+ const meta = JSON.parse(readFileSync(metaJsonPath, 'utf-8'));
210
+ const questionsToEmbed: string[] = [];
211
+
212
+ if (meta.question) {
213
+ questionsToEmbed.push(meta.question);
214
+ }
215
+ if (meta.improvedQuestion) {
216
+ questionsToEmbed.push(meta.improvedQuestion);
217
+ }
218
+
219
+ for (const q of questionsToEmbed) {
220
+ totalQuestions++;
221
+ const hash = hasher.h32ToString(q);
222
+ const cachePath = join(cacheDir, `${hash}.json`);
223
+
224
+ if (existsSync(cachePath)) {
225
+ questionsCached++;
226
+ } else {
227
+ // Embed the question
228
+ const { embeddings } = await embedMany({
229
+ model: fastembed.small,
230
+ values: [q],
231
+ });
232
+ try {
233
+ writeFileSync(cachePath, JSON.stringify(embeddings[0]));
234
+ questionsNew++;
235
+ } catch {
236
+ errorCount++;
237
+ }
238
+ }
239
+ }
240
+
241
+ questionStats = `, ${questionsToEmbed.length} questions`;
242
+ }
243
+
244
+ spinner.succeed(
245
+ `${progress} ${questionId}: ${lines.length} obs ` +
246
+ `(${uncachedLines.length} new, ${lines.length - uncachedLines.length} cached)${questionStats}`,
247
+ );
248
+ } catch (err) {
249
+ spinner.fail(`${progress} ${questionId}: Error - ${err}`);
250
+ errorCount++;
251
+ }
252
+
253
+ // Cooldown between questions to prevent CPU overload
254
+ if (cooldown > 0 && i < selectedQuestions.length - 1) {
255
+ await new Promise(resolve => setTimeout(resolve, cooldown));
256
+ }
257
+ }
258
+
259
+ // Summary
260
+ console.log('\n' + chalk.bold('Summary:'));
261
+ console.log(` Observations: ${totalObservations} total (${cachedCount} cached, ${newCount} new)`);
262
+ console.log(` Questions: ${totalQuestions} total (${questionsCached} cached, ${questionsNew} new)`);
263
+ if (errorCount > 0) {
264
+ console.log(chalk.red(` Errors: ${errorCount}`));
265
+ }
266
+ console.log(`\nCache location: ${cacheDir}`);
267
+ }
268
+
269
+ /**
270
+ * Parse observation lines from the observations block
271
+ * (Simplified version - just extracts non-header lines)
272
+ */
273
+ private parseObservationLines(observations: string): string[] {
274
+ const lines: string[] = [];
275
+
276
+ for (const line of observations.split('\n')) {
277
+ const trimmed = line.trim();
278
+
279
+ // Skip empty lines
280
+ if (!trimmed) continue;
281
+
282
+ // Skip date headers
283
+ if (/^##\s+\d{4}-\d{2}-\d{2}/.test(trimmed)) continue;
284
+ if (/^Date:\s+/i.test(trimmed)) continue;
285
+
286
+ // Skip thread XML tags
287
+ if (/^<thread\s+id=/.test(trimmed)) continue;
288
+ if (/^<\/thread>/.test(trimmed)) continue;
289
+ if (/^<other-conversation\s+id=/.test(trimmed)) continue;
290
+ if (/^<\/other-conversation>/.test(trimmed)) continue;
291
+
292
+ // Skip pattern headers
293
+ if (/^<patterns>/.test(trimmed)) continue;
294
+ if (/^<\/patterns>/.test(trimmed)) continue;
295
+
296
+ // This is an observation line
297
+ lines.push(trimmed);
298
+ }
299
+
300
+ return lines;
301
+ }
302
+ }