@mastra/longmemeval 0.0.0-error-handler-fix-20251020202607 → 0.0.0-execa-dynamic-import-20260304221256
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +847 -6
- package/LICENSE.md +15 -0
- package/docs/INVESTIGATION_WORKFLOW.md +314 -0
- package/package.json +34 -26
- package/scripts/download.ts +22 -1
- package/scripts/generate-wm-templates.ts +2 -1
- package/src/cli.ts +967 -155
- package/src/commands/__tests__/prepare.test.ts +11 -6
- package/src/commands/__tests__/run.test.ts +10 -8
- package/src/commands/clean.ts +147 -0
- package/src/commands/deterministic-ids.ts +126 -0
- package/src/commands/find-prohibited.ts +299 -0
- package/src/commands/investigate.ts +2437 -0
- package/src/commands/list-partial.ts +137 -0
- package/src/commands/obscure-thread-ids.ts +237 -0
- package/src/commands/partial-results.ts +198 -0
- package/src/commands/precompute-embeddings.ts +302 -0
- package/src/commands/prepare.ts +732 -105
- package/src/commands/reconcile.ts +383 -0
- package/src/commands/run.ts +1240 -121
- package/src/commands/sessions.ts +126 -0
- package/src/commands/sync.ts +152 -0
- package/src/commands/test-cleaned.ts +53 -0
- package/src/commands/test-incremental.ts +120 -0
- package/src/commands/test-messages.ts +95 -0
- package/src/commands/test-prohibited.ts +244 -0
- package/src/commands/test-thread.ts +136 -0
- package/src/commands/tokens.ts +625 -0
- package/src/config.ts +1253 -72
- package/src/data/types.ts +48 -6
- package/src/embeddings/cached-openai-embedding-model.ts +0 -1
- package/src/evaluation/longmemeval-metric.ts +186 -103
- package/src/processors/content-sanitizer.ts +153 -0
- package/src/processors/date-injector.ts +111 -0
- package/src/processors/index.ts +3 -0
- package/src/processors/observation-semantic-filter.ts +1142 -0
- package/src/retry-model.ts +5 -3
- package/src/storage/__tests__/benchmark-store.test.ts +18 -35
- package/src/storage/benchmark-store.ts +140 -153
- package/src/storage/benchmark-vector.ts +1 -1
- package/src/storage/index.ts +1 -0
- package/src/storage/persistable-inmemory.ts +133 -0
- package/src/test-utils/mock-model.ts +35 -0
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Precompute embeddings for all prepared questions
|
|
3
|
+
*
|
|
4
|
+
* This command indexes all observations from prepared data and caches
|
|
5
|
+
* the embeddings to disk. This makes subsequent benchmark runs much faster
|
|
6
|
+
* since embeddings don't need to be computed at runtime.
|
|
7
|
+
*
|
|
8
|
+
* Usage:
|
|
9
|
+
* pnpm precompute-embeddings -d longmemeval_s -c observational-memory
|
|
10
|
+
* pnpm precompute-embeddings -d longmemeval_s -c observational-memory --offset 100 --subset 50
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { existsSync, readdirSync, readFileSync, mkdirSync } from 'fs';
|
|
14
|
+
import { join } from 'path';
|
|
15
|
+
import ora from 'ora';
|
|
16
|
+
import chalk from 'chalk';
|
|
17
|
+
import xxhash from 'xxhash-wasm';
|
|
18
|
+
import { embedMany } from 'ai';
|
|
19
|
+
import { fastembed } from '@mastra/fastembed';
|
|
20
|
+
import { writeFileSync } from 'fs';
|
|
21
|
+
|
|
22
|
+
import type { DatasetType, MemoryConfigType } from '../data/types';
|
|
23
|
+
import { getMemoryConfig } from '../config';
|
|
24
|
+
|
|
25
|
+
export interface PrecomputeEmbeddingsOptions {
|
|
26
|
+
dataset: DatasetType;
|
|
27
|
+
memoryConfig: MemoryConfigType;
|
|
28
|
+
preparedDataDir?: string;
|
|
29
|
+
offset?: number;
|
|
30
|
+
subset?: number;
|
|
31
|
+
batchSize?: number;
|
|
32
|
+
cooldown?: number; // Cooldown in ms between questions
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
interface ObservationalMemoryRecord {
|
|
36
|
+
activeObservations?: string;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
interface ObservationalMemoryData {
|
|
40
|
+
// New format: observationalMemory is an array of [key, [record, ...]] tuples
|
|
41
|
+
// The second element is itself an array containing the record object(s)
|
|
42
|
+
observationalMemory?: Array<[string, ObservationalMemoryRecord[]]>;
|
|
43
|
+
// Old format: direct record object
|
|
44
|
+
record?: ObservationalMemoryRecord;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export class PrecomputeEmbeddingsCommand {
|
|
48
|
+
private hasher = xxhash();
|
|
49
|
+
|
|
50
|
+
async run(options: PrecomputeEmbeddingsOptions): Promise<void> {
|
|
51
|
+
const spinner = ora('Loading prepared data...').start();
|
|
52
|
+
|
|
53
|
+
const configDef = getMemoryConfig(options.memoryConfig);
|
|
54
|
+
|
|
55
|
+
// Determine which config to read from (handle readOnlyConfig)
|
|
56
|
+
const effectiveConfig =
|
|
57
|
+
configDef.readOnlyConfig && configDef.baseConfig ? configDef.baseConfig : options.memoryConfig;
|
|
58
|
+
|
|
59
|
+
const baseDir = options.preparedDataDir || join(process.cwd(), 'prepared-data');
|
|
60
|
+
const preparedDir = join(baseDir, options.dataset, effectiveConfig);
|
|
61
|
+
|
|
62
|
+
if (!existsSync(preparedDir)) {
|
|
63
|
+
spinner.fail(`Prepared data directory not found: ${preparedDir}`);
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// Find all question directories
|
|
68
|
+
const questionDirs = readdirSync(preparedDir, { withFileTypes: true })
|
|
69
|
+
.filter(d => d.isDirectory() && !d.name.startsWith('.'))
|
|
70
|
+
.map(d => d.name);
|
|
71
|
+
|
|
72
|
+
if (questionDirs.length === 0) {
|
|
73
|
+
spinner.fail('No prepared questions found');
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Apply offset and subset
|
|
78
|
+
let selectedQuestions = questionDirs;
|
|
79
|
+
if (options.offset) {
|
|
80
|
+
selectedQuestions = selectedQuestions.slice(options.offset);
|
|
81
|
+
}
|
|
82
|
+
if (options.subset) {
|
|
83
|
+
selectedQuestions = selectedQuestions.slice(0, options.subset);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
spinner.succeed(`Found ${questionDirs.length} questions, processing ${selectedQuestions.length}`);
|
|
87
|
+
|
|
88
|
+
// Create shared cache directory with model subdirectory
|
|
89
|
+
const modelName = 'fastembed-small';
|
|
90
|
+
const cacheDir = join(preparedDir, '..', '.embeddings-cache', modelName);
|
|
91
|
+
if (!existsSync(cacheDir)) {
|
|
92
|
+
mkdirSync(cacheDir, { recursive: true });
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
console.log(chalk.gray(`Cache directory: ${cacheDir}`));
|
|
96
|
+
console.log(chalk.gray(`Embedding model: ${modelName}`));
|
|
97
|
+
|
|
98
|
+
// Track stats
|
|
99
|
+
let totalObservations = 0;
|
|
100
|
+
let totalQuestions = 0;
|
|
101
|
+
let cachedCount = 0;
|
|
102
|
+
let newCount = 0;
|
|
103
|
+
let questionsCached = 0;
|
|
104
|
+
let questionsNew = 0;
|
|
105
|
+
let errorCount = 0;
|
|
106
|
+
|
|
107
|
+
const hasher = await this.hasher;
|
|
108
|
+
const batchSize = options.batchSize || 25; // Small batches to reduce CPU spikes
|
|
109
|
+
const cooldown = options.cooldown ?? 1000; // Default 1 second between questions
|
|
110
|
+
|
|
111
|
+
// Process each question
|
|
112
|
+
for (let i = 0; i < selectedQuestions.length; i++) {
|
|
113
|
+
const questionId = selectedQuestions[i];
|
|
114
|
+
const questionDir = join(preparedDir, questionId);
|
|
115
|
+
const omJsonPath = join(questionDir, 'om.json');
|
|
116
|
+
|
|
117
|
+
const progress = `[${i + 1}/${selectedQuestions.length}]`;
|
|
118
|
+
spinner.start(`${progress} Processing ${questionId}...`);
|
|
119
|
+
|
|
120
|
+
if (!existsSync(omJsonPath)) {
|
|
121
|
+
spinner.warn(`${progress} ${questionId}: No om.json found, skipping`);
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
try {
|
|
126
|
+
// Load observations from om.json
|
|
127
|
+
const omData: ObservationalMemoryData = JSON.parse(readFileSync(omJsonPath, 'utf-8'));
|
|
128
|
+
|
|
129
|
+
// Extract observations - handle both new format (array of tuples) and old format
|
|
130
|
+
let observations = '';
|
|
131
|
+
if (omData.observationalMemory && Array.isArray(omData.observationalMemory)) {
|
|
132
|
+
// New format: [[key, [record, ...]], ...] - get activeObservations from first record in first tuple
|
|
133
|
+
// Structure: observationalMemory[0] = [id, [record_object, ...]]
|
|
134
|
+
const recordArray = omData.observationalMemory[0]?.[1];
|
|
135
|
+
const firstRecord = Array.isArray(recordArray) ? recordArray[0] : undefined;
|
|
136
|
+
observations = firstRecord?.activeObservations || '';
|
|
137
|
+
} else if (omData.record?.activeObservations) {
|
|
138
|
+
// Old format: direct record object
|
|
139
|
+
observations = omData.record.activeObservations;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
if (!observations) {
|
|
143
|
+
spinner.info(`${progress} ${questionId}: No observations found`);
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// Parse observation lines
|
|
148
|
+
const lines = this.parseObservationLines(observations);
|
|
149
|
+
totalObservations += lines.length;
|
|
150
|
+
|
|
151
|
+
// Check which lines need embedding
|
|
152
|
+
const uncachedLines: { line: string; hash: string }[] = [];
|
|
153
|
+
|
|
154
|
+
for (const line of lines) {
|
|
155
|
+
const hash = hasher.h32ToString(line);
|
|
156
|
+
const cachePath = join(cacheDir, `${hash}.json`);
|
|
157
|
+
|
|
158
|
+
if (existsSync(cachePath)) {
|
|
159
|
+
cachedCount++;
|
|
160
|
+
} else {
|
|
161
|
+
uncachedLines.push({ line, hash });
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
if (uncachedLines.length === 0) {
|
|
166
|
+
spinner.succeed(`${progress} ${questionId}: ${lines.length} observations (all cached)`);
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
// Embed in batches with micro-cooldowns to prevent CPU overload
|
|
171
|
+
const batchCooldown = Math.min(cooldown / 4, 250); // 250ms max between batches
|
|
172
|
+
const totalBatches = Math.ceil(uncachedLines.length / batchSize);
|
|
173
|
+
|
|
174
|
+
for (let b = 0; b < uncachedLines.length; b += batchSize) {
|
|
175
|
+
const batchNum = Math.floor(b / batchSize) + 1;
|
|
176
|
+
const batch = uncachedLines.slice(b, b + batchSize);
|
|
177
|
+
|
|
178
|
+
spinner.text = `${progress} ${questionId}: embedding batch ${batchNum}/${totalBatches} (${batch.length} items)`;
|
|
179
|
+
|
|
180
|
+
const { embeddings } = await embedMany({
|
|
181
|
+
model: fastembed.small,
|
|
182
|
+
values: batch.map(l => l.line),
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
// Save to cache
|
|
186
|
+
for (let j = 0; j < batch.length; j++) {
|
|
187
|
+
const { hash } = batch[j];
|
|
188
|
+
const embedding = embeddings[j];
|
|
189
|
+
const cachePath = join(cacheDir, `${hash}.json`);
|
|
190
|
+
|
|
191
|
+
try {
|
|
192
|
+
writeFileSync(cachePath, JSON.stringify(embedding));
|
|
193
|
+
newCount++;
|
|
194
|
+
} catch {
|
|
195
|
+
errorCount++;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// Micro-cooldown between batches within a question
|
|
200
|
+
if (batchCooldown > 0 && b + batchSize < uncachedLines.length) {
|
|
201
|
+
await new Promise(resolve => setTimeout(resolve, batchCooldown));
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// Also embed the question(s) from meta.json
|
|
206
|
+
const metaJsonPath = join(questionDir, 'meta.json');
|
|
207
|
+
let questionStats = '';
|
|
208
|
+
if (existsSync(metaJsonPath)) {
|
|
209
|
+
const meta = JSON.parse(readFileSync(metaJsonPath, 'utf-8'));
|
|
210
|
+
const questionsToEmbed: string[] = [];
|
|
211
|
+
|
|
212
|
+
if (meta.question) {
|
|
213
|
+
questionsToEmbed.push(meta.question);
|
|
214
|
+
}
|
|
215
|
+
if (meta.improvedQuestion) {
|
|
216
|
+
questionsToEmbed.push(meta.improvedQuestion);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
for (const q of questionsToEmbed) {
|
|
220
|
+
totalQuestions++;
|
|
221
|
+
const hash = hasher.h32ToString(q);
|
|
222
|
+
const cachePath = join(cacheDir, `${hash}.json`);
|
|
223
|
+
|
|
224
|
+
if (existsSync(cachePath)) {
|
|
225
|
+
questionsCached++;
|
|
226
|
+
} else {
|
|
227
|
+
// Embed the question
|
|
228
|
+
const { embeddings } = await embedMany({
|
|
229
|
+
model: fastembed.small,
|
|
230
|
+
values: [q],
|
|
231
|
+
});
|
|
232
|
+
try {
|
|
233
|
+
writeFileSync(cachePath, JSON.stringify(embeddings[0]));
|
|
234
|
+
questionsNew++;
|
|
235
|
+
} catch {
|
|
236
|
+
errorCount++;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
questionStats = `, ${questionsToEmbed.length} questions`;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
spinner.succeed(
|
|
245
|
+
`${progress} ${questionId}: ${lines.length} obs ` +
|
|
246
|
+
`(${uncachedLines.length} new, ${lines.length - uncachedLines.length} cached)${questionStats}`,
|
|
247
|
+
);
|
|
248
|
+
} catch (err) {
|
|
249
|
+
spinner.fail(`${progress} ${questionId}: Error - ${err}`);
|
|
250
|
+
errorCount++;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// Cooldown between questions to prevent CPU overload
|
|
254
|
+
if (cooldown > 0 && i < selectedQuestions.length - 1) {
|
|
255
|
+
await new Promise(resolve => setTimeout(resolve, cooldown));
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
// Summary
|
|
260
|
+
console.log('\n' + chalk.bold('Summary:'));
|
|
261
|
+
console.log(` Observations: ${totalObservations} total (${cachedCount} cached, ${newCount} new)`);
|
|
262
|
+
console.log(` Questions: ${totalQuestions} total (${questionsCached} cached, ${questionsNew} new)`);
|
|
263
|
+
if (errorCount > 0) {
|
|
264
|
+
console.log(chalk.red(` Errors: ${errorCount}`));
|
|
265
|
+
}
|
|
266
|
+
console.log(`\nCache location: ${cacheDir}`);
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Parse observation lines from the observations block
|
|
271
|
+
* (Simplified version - just extracts non-header lines)
|
|
272
|
+
*/
|
|
273
|
+
private parseObservationLines(observations: string): string[] {
|
|
274
|
+
const lines: string[] = [];
|
|
275
|
+
|
|
276
|
+
for (const line of observations.split('\n')) {
|
|
277
|
+
const trimmed = line.trim();
|
|
278
|
+
|
|
279
|
+
// Skip empty lines
|
|
280
|
+
if (!trimmed) continue;
|
|
281
|
+
|
|
282
|
+
// Skip date headers
|
|
283
|
+
if (/^##\s+\d{4}-\d{2}-\d{2}/.test(trimmed)) continue;
|
|
284
|
+
if (/^Date:\s+/i.test(trimmed)) continue;
|
|
285
|
+
|
|
286
|
+
// Skip thread XML tags
|
|
287
|
+
if (/^<thread\s+id=/.test(trimmed)) continue;
|
|
288
|
+
if (/^<\/thread>/.test(trimmed)) continue;
|
|
289
|
+
if (/^<other-conversation\s+id=/.test(trimmed)) continue;
|
|
290
|
+
if (/^<\/other-conversation>/.test(trimmed)) continue;
|
|
291
|
+
|
|
292
|
+
// Skip pattern headers
|
|
293
|
+
if (/^<patterns>/.test(trimmed)) continue;
|
|
294
|
+
if (/^<\/patterns>/.test(trimmed)) continue;
|
|
295
|
+
|
|
296
|
+
// This is an observation line
|
|
297
|
+
lines.push(trimmed);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
return lines;
|
|
301
|
+
}
|
|
302
|
+
}
|