monomind 1.18.9 → 1.18.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/package.json +1 -1
  2. package/packages/@monomind/cli/dist/src/commands/hooks-coverage-commands.d.ts +1 -2
  3. package/packages/@monomind/cli/dist/src/commands/hooks-coverage-commands.js +1 -2
  4. package/packages/@monomind/cli/dist/src/commands/hooks-coverage-gaps.d.ts +1 -2
  5. package/packages/@monomind/cli/dist/src/commands/hooks-coverage-gaps.js +2 -117
  6. package/packages/@monomind/cli/dist/src/commands/hooks.js +1 -2
  7. package/packages/@monomind/cli/dist/src/commands/index.d.ts +0 -1
  8. package/packages/@monomind/cli/dist/src/commands/index.js +0 -1
  9. package/packages/@monomind/cli/dist/src/init/executor.d.ts +3 -0
  10. package/packages/@monomind/cli/dist/src/init/executor.js +20 -111
  11. package/packages/@monomind/cli/dist/src/mcp-client.js +0 -2
  12. package/packages/@monomind/cli/dist/src/mcp-tools/index.d.ts +0 -1
  13. package/packages/@monomind/cli/dist/src/mcp-tools/index.js +0 -1
  14. package/packages/@monomind/cli/dist/src/mcp-tools/quality/coverage-analysis/track-trends.d.ts +4 -4
  15. package/packages/@monomind/cli/dist/src/mcp-tools/quality/test-generation/suggest-tests.d.ts +4 -4
  16. package/packages/@monomind/cli/package.json +1 -1
  17. package/packages/@monomind/cli/dist/src/benchmarks/benchmark-runner.d.ts +0 -134
  18. package/packages/@monomind/cli/dist/src/benchmarks/benchmark-runner.js +0 -340
  19. package/packages/@monomind/cli/dist/src/benchmarks/metric-evaluators.d.ts +0 -36
  20. package/packages/@monomind/cli/dist/src/benchmarks/metric-evaluators.js +0 -125
  21. package/packages/@monomind/cli/dist/src/commands/benchmark.d.ts +0 -10
  22. package/packages/@monomind/cli/dist/src/commands/benchmark.js +0 -586
  23. package/packages/@monomind/cli/dist/src/commands/migrate.d.ts +0 -8
  24. package/packages/@monomind/cli/dist/src/commands/migrate.js +0 -789
  25. package/packages/@monomind/cli/dist/src/commands/monovector/backup.d.ts +0 -11
  26. package/packages/@monomind/cli/dist/src/commands/monovector/backup.js +0 -752
  27. package/packages/@monomind/cli/dist/src/commands/monovector/benchmark.d.ts +0 -11
  28. package/packages/@monomind/cli/dist/src/commands/monovector/benchmark.js +0 -502
  29. package/packages/@monomind/cli/dist/src/commands/monovector/import.d.ts +0 -18
  30. package/packages/@monomind/cli/dist/src/commands/monovector/import.js +0 -374
  31. package/packages/@monomind/cli/dist/src/commands/monovector/index.d.ts +0 -29
  32. package/packages/@monomind/cli/dist/src/commands/monovector/index.js +0 -129
  33. package/packages/@monomind/cli/dist/src/commands/monovector/init.d.ts +0 -11
  34. package/packages/@monomind/cli/dist/src/commands/monovector/init.js +0 -481
  35. package/packages/@monomind/cli/dist/src/commands/monovector/migrate.d.ts +0 -11
  36. package/packages/@monomind/cli/dist/src/commands/monovector/migrate.js +0 -500
  37. package/packages/@monomind/cli/dist/src/commands/monovector/optimize.d.ts +0 -11
  38. package/packages/@monomind/cli/dist/src/commands/monovector/optimize.js +0 -515
  39. package/packages/@monomind/cli/dist/src/commands/monovector/setup.d.ts +0 -18
  40. package/packages/@monomind/cli/dist/src/commands/monovector/setup.js +0 -775
  41. package/packages/@monomind/cli/dist/src/commands/monovector/status.d.ts +0 -11
  42. package/packages/@monomind/cli/dist/src/commands/monovector/status.js +0 -491
  43. package/packages/@monomind/cli/dist/src/commands/progress.d.ts +0 -11
  44. package/packages/@monomind/cli/dist/src/commands/progress.js +0 -261
  45. package/packages/@monomind/cli/dist/src/mcp-tools/progress-tools.d.ts +0 -14
  46. package/packages/@monomind/cli/dist/src/mcp-tools/progress-tools.js +0 -353
@@ -1,586 +0,0 @@
1
- /**
2
- * CLI Benchmark Command
3
- * Comprehensive benchmarking for self-learning, pre-training, and neural systems
4
- *
5
- * @module v1/cli/commands/benchmark
6
- */
7
- import { output } from '../output.js';
8
- import { writeFileSync, renameSync, readFileSync, existsSync, mkdirSync, statSync } from 'node:fs';
9
- import { join } from 'node:path';
10
- import { BenchmarkRunner } from '../benchmarks/benchmark-runner.js';
11
- // ============================================================================
12
- // Pretrain Benchmark Subcommand
13
- // ============================================================================
14
- const pretrainCommand = {
15
- name: 'pretrain',
16
- description: 'Benchmark pattern-learning pre-training system',
17
- options: [
18
- { name: 'iterations', short: 'i', type: 'number', description: 'Benchmark iterations', default: '100' },
19
- { name: 'warmup', short: 'w', type: 'number', description: 'Warmup iterations', default: '10' },
20
- { name: 'output', short: 'o', type: 'string', description: 'Output format: text, json', default: 'text' },
21
- { name: 'save', short: 's', type: 'string', description: 'Save results to file' },
22
- { name: 'verbose', short: 'v', type: 'boolean', description: 'Verbose output', default: 'false' },
23
- ],
24
- examples: [
25
- { command: 'monomind benchmark pretrain', description: 'Run pre-training benchmarks' },
26
- { command: 'monomind benchmark pretrain -i 500 --save results.json', description: 'Extended benchmark with results saved' },
27
- { command: 'monomind benchmark pretrain -o json', description: 'Output results as JSON' },
28
- ],
29
- action: async (ctx) => {
30
- const iterations = parseInt(ctx.flags.iterations || '100', 10);
31
- const warmup = parseInt(ctx.flags.warmup || '10', 10);
32
- const outputFormat = ctx.flags.output || 'text';
33
- const saveFile = ctx.flags.save;
34
- const verbose = ctx.flags.verbose === true;
35
- void iterations;
36
- void warmup;
37
- void verbose;
38
- void outputFormat;
39
- void saveFile;
40
- output.writeln();
41
- output.writeln(output.bold('Pattern-Learning Benchmark'));
42
- output.writeln(output.dim('─'.repeat(50)));
43
- output.writeln(output.dim(' The native SONA/EWC++/MoE pre-training benchmark was removed in the lean build.'));
44
- output.writeln(output.dim(' Use "monomind benchmark neural" to benchmark the active pure-JS intelligence layer.'));
45
- output.writeln();
46
- return { success: true };
47
- },
48
- };
49
- // ============================================================================
50
- // Neural Benchmark Subcommand
51
- // ============================================================================
52
- const neuralCommand = {
53
- name: 'neural',
54
- description: 'Benchmark neural operations (embeddings, HNSW, cosine similarity)',
55
- options: [
56
- { name: 'iterations', short: 'i', type: 'number', description: 'Benchmark iterations', default: '100' },
57
- { name: 'dimension', short: 'd', type: 'number', description: 'Embedding dimension', default: '384' },
58
- { name: 'vectors', short: 'n', type: 'number', description: 'Number of test vectors', default: '1000' },
59
- { name: 'output', short: 'o', type: 'string', description: 'Output format: text, json', default: 'text' },
60
- ],
61
- examples: [
62
- { command: 'monomind benchmark neural', description: 'Run neural benchmarks' },
63
- { command: 'monomind benchmark neural -d 768 -n 5000', description: 'Higher dimension, more vectors' },
64
- ],
65
- action: async (ctx) => {
66
- const iterationsRaw = parseInt(ctx.flags.iterations || '100', 10);
67
- const iterations = Number.isFinite(iterationsRaw) ? Math.max(1, Math.min(iterationsRaw, 10_000)) : 100;
68
- const dimensionRaw = parseInt(ctx.flags.dimension || '384', 10);
69
- const dimension = Number.isFinite(dimensionRaw) ? Math.max(1, Math.min(dimensionRaw, 4096)) : 384;
70
- const numVectorsRaw = parseInt(ctx.flags.vectors || '1000', 10);
71
- const numVectors = Number.isFinite(numVectorsRaw) ? Math.max(1, Math.min(numVectorsRaw, 100_000)) : 1000;
72
- const outputFormat = ctx.flags.output || 'text';
73
- output.writeln();
74
- output.writeln(output.bold('Neural Operations Benchmark'));
75
- output.writeln(output.dim('─'.repeat(60)));
76
- output.writeln(`Iterations: ${iterations} | Dimension: ${dimension} | Vectors: ${numVectors}`);
77
- output.writeln();
78
- const spinner = output.createSpinner({ text: 'Running neural benchmarks...', spinner: 'dots' });
79
- spinner.start();
80
- try {
81
- const { performance } = await import('node:perf_hooks');
82
- // Helper functions
83
- const percentile = (sorted, p) => {
84
- const idx = Math.ceil((p / 100) * sorted.length) - 1;
85
- return sorted[Math.max(0, idx)];
86
- };
87
- const results = [];
88
- // 1. Embedding Generation
89
- spinner.setText('Benchmarking embedding generation...');
90
- let generateEmbedding;
91
- try {
92
- const memory = await import('../memory/memory-initializer.js');
93
- generateEmbedding = memory.generateEmbedding;
94
- }
95
- catch {
96
- generateEmbedding = async (text) => {
97
- const emb = [];
98
- for (let i = 0; i < dimension; i++) {
99
- emb.push(Math.sin(text.charCodeAt(i % text.length) * (i + 1)));
100
- }
101
- return { embedding: emb, dimensions: dimension, model: 'fallback' };
102
- };
103
- }
104
- const embedTimes = [];
105
- for (let i = 0; i < iterations; i++) {
106
- const start = performance.now();
107
- await generateEmbedding(`benchmark text ${i}`);
108
- embedTimes.push(performance.now() - start);
109
- }
110
- const embedMean = embedTimes.reduce((a, b) => a + b, 0) / embedTimes.length;
111
- const embedSorted = [...embedTimes].sort((a, b) => a - b);
112
- results.push({
113
- name: 'Embedding Generation',
114
- mean: embedMean,
115
- p95: percentile(embedSorted, 95),
116
- p99: percentile(embedSorted, 99),
117
- target: 5.0,
118
- met: embedMean <= 5.0,
119
- });
120
- // 2. Batch Cosine Similarity
121
- spinner.setText('Benchmarking batch cosine similarity...');
122
- let batchCosineSim;
123
- try {
124
- const memory = await import('../memory/memory-initializer.js');
125
- batchCosineSim = memory.batchCosineSim;
126
- }
127
- catch {
128
- batchCosineSim = (query, vectors) => {
129
- const res = new Float32Array(vectors.length);
130
- for (let i = 0; i < vectors.length; i++) {
131
- let dot = 0, nQ = 0, nV = 0;
132
- for (let j = 0; j < query.length; j++) {
133
- dot += query[j] * vectors[i][j];
134
- nQ += query[j] * query[j];
135
- nV += vectors[i][j] * vectors[i][j];
136
- }
137
- res[i] = dot / (Math.sqrt(nQ) * Math.sqrt(nV));
138
- }
139
- return res;
140
- };
141
- }
142
- const query = new Float32Array(dimension).map(() => Math.random());
143
- const vectors = Array.from({ length: numVectors }, () => new Float32Array(dimension).map(() => Math.random()));
144
- const cosineTimes = [];
145
- for (let i = 0; i < Math.min(iterations, 50); i++) {
146
- const start = performance.now();
147
- batchCosineSim(query, vectors);
148
- cosineTimes.push(performance.now() - start);
149
- }
150
- const cosineMean = cosineTimes.reduce((a, b) => a + b, 0) / cosineTimes.length;
151
- const cosineSorted = [...cosineTimes].sort((a, b) => a - b);
152
- results.push({
153
- name: `Batch Cosine (${numVectors} vectors)`,
154
- mean: cosineMean,
155
- p95: percentile(cosineSorted, 95),
156
- p99: percentile(cosineSorted, 99),
157
- target: 5.0,
158
- met: cosineMean <= 5.0,
159
- });
160
- // 3. Softmax-weighted top-K search (pure-JS)
161
- spinner.setText('Benchmarking softmax top-K search...');
162
- const flashTimes = [];
163
- try {
164
- const memory = await import('../memory/memory-initializer.js');
165
- if (memory.flashAttentionSearch) {
166
- for (let i = 0; i < Math.min(iterations, 50); i++) {
167
- const start = performance.now();
168
- memory.flashAttentionSearch(query, vectors, { k: 10 });
169
- flashTimes.push(performance.now() - start);
170
- }
171
- }
172
- }
173
- catch {
174
- // not available
175
- }
176
- if (flashTimes.length > 0) {
177
- const flashMean = flashTimes.reduce((a, b) => a + b, 0) / flashTimes.length;
178
- const flashSorted = [...flashTimes].sort((a, b) => a - b);
179
- results.push({
180
- name: 'Softmax Top-K Search',
181
- mean: flashMean,
182
- p95: percentile(flashSorted, 95),
183
- p99: percentile(flashSorted, 99),
184
- target: 2.0,
185
- met: flashMean <= 2.0,
186
- });
187
- }
188
- spinner.stop();
189
- // Display results
190
- output.writeln();
191
- output.writeln(output.bold('Results'));
192
- output.writeln(output.dim('─'.repeat(60)));
193
- for (const r of results) {
194
- const status = r.met ? output.success('✓') : output.error('✗');
195
- output.writeln(`${status} ${r.name}`);
196
- output.writeln(` Mean: ${r.mean.toFixed(3)}ms | p95: ${r.p95.toFixed(3)}ms | p99: ${r.p99.toFixed(3)}ms`);
197
- output.writeln(` Target: ${r.target}ms | Status: ${r.met ? 'Met' : 'Not met'}`);
198
- output.writeln();
199
- }
200
- if (outputFormat === 'json') {
201
- output.writeln(JSON.stringify(results, null, 2));
202
- }
203
- const allPassed = results.every(r => r.met);
204
- return {
205
- success: true,
206
- message: allPassed ? 'All neural benchmarks passed!' : 'Some benchmarks below target',
207
- };
208
- }
209
- catch (err) {
210
- spinner.stop();
211
- const errorMsg = err instanceof Error ? err.message : String(err);
212
- output.writeln(output.error(`Neural benchmark failed: ${errorMsg}`));
213
- return {
214
- success: false,
215
- message: `Neural benchmark failed: ${errorMsg}`,
216
- };
217
- }
218
- },
219
- };
220
- // ============================================================================
221
- // Memory Benchmark Subcommand
222
- // ============================================================================
223
- const memoryCommand = {
224
- name: 'memory',
225
- description: 'Benchmark memory operations (HNSW search, store, retrieve)',
226
- options: [
227
- { name: 'iterations', short: 'i', type: 'number', description: 'Benchmark iterations', default: '100' },
228
- { name: 'output', short: 'o', type: 'string', description: 'Output format: text, json', default: 'text' },
229
- ],
230
- examples: [
231
- { command: 'monomind benchmark memory', description: 'Run memory benchmarks' },
232
- ],
233
- action: async (ctx) => {
234
- const iterationsRaw = parseInt(ctx.flags.iterations || '100', 10);
235
- const iterations = Number.isFinite(iterationsRaw) ? Math.max(1, Math.min(iterationsRaw, 10_000)) : 100;
236
- const outputFormat = ctx.flags.output || 'text';
237
- output.writeln();
238
- output.writeln(output.bold('Memory Operations Benchmark'));
239
- output.writeln(output.dim('─'.repeat(60)));
240
- const spinner = output.createSpinner({ text: 'Running memory benchmarks...', spinner: 'dots' });
241
- spinner.start();
242
- try {
243
- const { performance } = await import('node:perf_hooks');
244
- const percentile = (sorted, p) => {
245
- const idx = Math.ceil((p / 100) * sorted.length) - 1;
246
- return sorted[Math.max(0, idx)];
247
- };
248
- const results = [];
249
- // Import memory functions
250
- let storeEntry;
251
- let searchEntries;
252
- try {
253
- const memory = await import('../memory/memory-initializer.js');
254
- storeEntry = memory.storeEntry;
255
- searchEntries = memory.searchEntries;
256
- }
257
- catch {
258
- // @monomind/memory not available — return null metrics instead of fake numbers
259
- storeEntry = async () => ({ success: true });
260
- searchEntries = async () => ({ results: [], searchTime: 0 }); // 0 = no-op fallback, not a real benchmark
261
- }
262
- // 1. Store benchmark
263
- spinner.setText('Benchmarking memory store...');
264
- const storeTimes = [];
265
- for (let i = 0; i < iterations; i++) {
266
- const start = performance.now();
267
- await storeEntry({
268
- key: `bench-key-${i}`,
269
- value: `Benchmark value ${i} with some additional content`,
270
- namespace: 'benchmark',
271
- });
272
- storeTimes.push(performance.now() - start);
273
- }
274
- const storeMean = storeTimes.reduce((a, b) => a + b, 0) / storeTimes.length;
275
- results.push({
276
- name: 'Memory Store',
277
- mean: storeMean,
278
- p95: percentile([...storeTimes].sort((a, b) => a - b), 95),
279
- target: 10.0,
280
- met: storeMean <= 10.0,
281
- });
282
- // 2. Search benchmark
283
- spinner.setText('Benchmarking memory search...');
284
- const queries = [
285
- 'authentication patterns',
286
- 'error handling best practices',
287
- 'performance optimization',
288
- 'testing strategies',
289
- 'security vulnerabilities',
290
- ];
291
- const searchTimes = [];
292
- for (let i = 0; i < iterations; i++) {
293
- const start = performance.now();
294
- await searchEntries({
295
- query: queries[i % queries.length],
296
- namespace: 'benchmark',
297
- limit: 10,
298
- });
299
- searchTimes.push(performance.now() - start);
300
- }
301
- const searchMean = searchTimes.reduce((a, b) => a + b, 0) / searchTimes.length;
302
- results.push({
303
- name: 'Memory Search (HNSW)',
304
- mean: searchMean,
305
- p95: percentile([...searchTimes].sort((a, b) => a - b), 95),
306
- target: 10.0,
307
- met: searchMean <= 10.0,
308
- });
309
- spinner.stop();
310
- // Display results
311
- output.writeln();
312
- output.writeln(output.bold('Results'));
313
- output.writeln(output.dim('─'.repeat(60)));
314
- for (const r of results) {
315
- const status = r.met ? output.success('✓') : output.error('✗');
316
- output.writeln(`${status} ${r.name}`);
317
- output.writeln(` Mean: ${r.mean.toFixed(3)}ms | p95: ${r.p95.toFixed(3)}ms | Target: ${r.target}ms`);
318
- output.writeln();
319
- }
320
- if (outputFormat === 'json') {
321
- output.writeln(JSON.stringify(results, null, 2));
322
- }
323
- return { success: true, message: 'Memory benchmarks complete' };
324
- }
325
- catch (err) {
326
- spinner.stop();
327
- const errorMsg = err instanceof Error ? err.message : String(err);
328
- output.writeln(output.error(`Memory benchmark failed: ${errorMsg}`));
329
- return {
330
- success: false,
331
- message: `Memory benchmark failed: ${errorMsg}`,
332
- };
333
- }
334
- },
335
- };
336
- // ============================================================================
337
- // Full Suite Benchmark Subcommand
338
- // ============================================================================
339
- const allCommand = {
340
- name: 'all',
341
- description: 'Run all benchmark suites',
342
- options: [
343
- { name: 'iterations', short: 'i', type: 'number', description: 'Benchmark iterations', default: '50' },
344
- { name: 'output', short: 'o', type: 'string', description: 'Output format: text, json', default: 'text' },
345
- { name: 'save', short: 's', type: 'string', description: 'Save results to file' },
346
- ],
347
- examples: [
348
- { command: 'monomind benchmark all', description: 'Run all benchmarks' },
349
- { command: 'monomind benchmark all --save full-results.json', description: 'Run all and save results' },
350
- ],
351
- action: async (ctx) => {
352
- output.writeln();
353
- output.writeln(output.bold(output.highlight('═'.repeat(65))));
354
- output.writeln(output.bold(' Monomind - Full Benchmark Suite'));
355
- output.writeln(output.bold(output.highlight('═'.repeat(65))));
356
- const startTime = Date.now();
357
- const allResults = {};
358
- // Run pretrain benchmarks
359
- output.writeln();
360
- output.writeln(output.bold('▸ Pre-Training Benchmarks'));
361
- if (pretrainCommand.action) {
362
- const pretrainResult = await pretrainCommand.action(ctx);
363
- allResults.pretrain = pretrainResult;
364
- }
365
- // Run neural benchmarks
366
- output.writeln();
367
- output.writeln(output.bold('▸ Neural Benchmarks'));
368
- if (neuralCommand.action) {
369
- const neuralResult = await neuralCommand.action(ctx);
370
- allResults.neural = neuralResult;
371
- }
372
- // Run memory benchmarks
373
- output.writeln();
374
- output.writeln(output.bold('▸ Memory Benchmarks'));
375
- if (memoryCommand.action) {
376
- const memoryResult = await memoryCommand.action(ctx);
377
- allResults.memory = memoryResult;
378
- }
379
- const totalDuration = Date.now() - startTime;
380
- output.writeln();
381
- output.writeln(output.bold(output.highlight('═'.repeat(65))));
382
- output.writeln(` Total Duration: ${(totalDuration / 1000).toFixed(2)}s`);
383
- output.writeln(output.bold(output.highlight('═'.repeat(65))));
384
- // Save if requested
385
- const saveFile = ctx.flags.save;
386
- if (saveFile) {
387
- const resultsDir = join(process.cwd(), '.monomind', 'benchmarks');
388
- if (!existsSync(resultsDir)) {
389
- mkdirSync(resultsDir, { recursive: true });
390
- }
391
- // Path traversal guard: resolve within resultsDir regardless of whether saveFile is absolute
392
- const { resolve: resolvePath, basename } = await import('node:path');
393
- const safeName = basename(saveFile);
394
- const savePath = resolvePath(resultsDir, safeName);
395
- const resolvedResultsDir = resolvePath(resultsDir);
396
- if (!savePath.startsWith(resolvedResultsDir + '/') && savePath !== resolvedResultsDir) {
397
- output.writeln(output.error(`Save path must be within ${resultsDir}`));
398
- return { success: false, message: 'Invalid save path' };
399
- }
400
- const saveTmp2 = savePath + '.tmp';
401
- writeFileSync(saveTmp2, JSON.stringify({
402
- timestamp: new Date().toISOString(),
403
- duration: totalDuration,
404
- results: allResults,
405
- }, null, 2));
406
- renameSync(saveTmp2, savePath);
407
- output.writeln(output.success(`Results saved to ${savePath}`));
408
- }
409
- return { success: true, message: 'All benchmarks complete' };
410
- },
411
- };
412
- // ============================================================================
413
- // Regression Benchmark Subcommand
414
- // ============================================================================
415
- const regressionCommand = {
416
- name: 'regression',
417
- description: 'Quality regression testing using benchmark definitions and baselines',
418
- options: [
419
- { name: 'suite', short: 's', type: 'string', description: 'Path to benchmark definitions directory', default: '.monomind/benchmarks/definitions' },
420
- { name: 'benchmark-id', short: 'b', type: 'string', description: 'Run a specific benchmark by ID' },
421
- { name: 'agent-output', short: 'a', type: 'string', description: 'Path to file containing agent output to evaluate' },
422
- { name: 'pin-baseline', type: 'boolean', description: 'Save current results as the new baseline', default: 'false' },
423
- { name: 'output', short: 'o', type: 'string', description: 'Output format: text, json', default: 'text' },
424
- ],
425
- examples: [
426
- { command: 'monomind benchmark regression', description: 'List all benchmark definitions' },
427
- { command: 'monomind benchmark regression -b agent-spawn -a output.txt', description: 'Evaluate agent output against a benchmark' },
428
- { command: 'monomind benchmark regression -b agent-spawn -a output.txt --pin-baseline', description: 'Evaluate and pin results as new baseline' },
429
- ],
430
- action: async (ctx) => {
431
- const suiteDirRaw = ctx.flags.suite || '.monomind/benchmarks/definitions';
432
- const benchmarkId = ctx.flags['benchmark-id'];
433
- const agentOutputFile = ctx.flags['agent-output'];
434
- const pinBaseline = ctx.flags['pin-baseline'] === true;
435
- const outputFormat = ctx.flags.output || 'text';
436
- // Validate benchmarkId to prevent path traversal in baseline file names
437
- if (benchmarkId !== undefined) {
438
- if (!/^[a-zA-Z0-9_-]{1,128}$/.test(benchmarkId)) {
439
- output.writeln(output.error('Invalid benchmark-id: must contain only alphanumeric, dash, or underscore characters (max 128).'));
440
- return { success: false, message: 'Invalid benchmark-id' };
441
- }
442
- }
443
- const runner = new BenchmarkRunner();
444
- const baselinesDir = join(process.cwd(), '.monomind', 'benchmarks', 'baselines');
445
- // Path traversal guard for suiteDir
446
- const { resolve: resolvePath2 } = await import('node:path');
447
- const projectRoot = resolvePath2(process.cwd());
448
- const resolvedSuiteDir = resolvePath2(process.cwd(), suiteDirRaw);
449
- if (!resolvedSuiteDir.startsWith(projectRoot + '/') && resolvedSuiteDir !== projectRoot) {
450
- output.writeln(output.error(`Suite directory must be within the project: ${projectRoot}`));
451
- return { success: false, message: 'Invalid suite directory' };
452
- }
453
- const suiteDir = suiteDirRaw;
454
- const definitions = runner.loadBenchmarks(resolvedSuiteDir);
455
- if (definitions.length === 0) {
456
- output.writeln(output.dim(`No benchmark definitions found in ${suiteDir}`));
457
- output.writeln(output.dim('Create JSON files there to define quality benchmarks.'));
458
- return { success: true, message: 'No benchmarks defined' };
459
- }
460
- // List mode — no agent output provided
461
- if (!agentOutputFile) {
462
- output.writeln();
463
- output.writeln(output.bold('Benchmark Definitions'));
464
- output.writeln(output.dim('─'.repeat(50)));
465
- for (const def of definitions) {
466
- output.writeln(` ${output.highlight(def.benchmarkId)} ${output.dim(def.agentSlug)} — ${def.qualityMetrics?.length ?? 0} metrics`);
467
- }
468
- output.writeln();
469
- output.writeln(output.dim('Use --agent-output <file> to evaluate against a benchmark.'));
470
- return { success: true, message: `${definitions.length} benchmarks loaded` };
471
- }
472
- // Evaluation mode
473
- if (!existsSync(agentOutputFile)) {
474
- output.writeln(output.error(`Agent output file not found: ${agentOutputFile}`));
475
- return { success: false, message: 'Agent output file not found' };
476
- }
477
- const MAX_AGENT_OUTPUT_BYTES = 10 * 1024 * 1024; // 10 MB
478
- try {
479
- const agentOutputStat = statSync(agentOutputFile);
480
- if (agentOutputStat.size > MAX_AGENT_OUTPUT_BYTES) {
481
- output.writeln(output.error(`Agent output file too large: ${agentOutputFile} (max 10 MB)`));
482
- return { success: false, message: 'Agent output file too large' };
483
- }
484
- }
485
- catch { /* existsSync already passed; ignore stat failure */ }
486
- const agentOutput = readFileSync(agentOutputFile, 'utf-8');
487
- const targetDefs = benchmarkId
488
- ? definitions.filter((d) => d.benchmarkId === benchmarkId)
489
- : definitions;
490
- if (targetDefs.length === 0) {
491
- output.writeln(output.error(`No benchmark found with id: ${benchmarkId}`));
492
- return { success: false, message: 'Benchmark not found' };
493
- }
494
- const results = targetDefs.map((def) => runner.runBenchmark(def, agentOutput));
495
- if (outputFormat === 'json') {
496
- output.writeln(JSON.stringify(results, null, 2));
497
- }
498
- else {
499
- output.writeln();
500
- output.writeln(output.bold('Regression Results'));
501
- output.writeln(output.dim('─'.repeat(50)));
502
- for (const result of results) {
503
- const status = result.passed ? output.success('PASS') : output.error('FAIL');
504
- output.writeln(` ${status} ${result.benchmarkId} ${output.dim(`${result.durationMs}ms`)}`);
505
- for (const m of result.metricResults) {
506
- const mStatus = m.passed ? ' ✓' : ' ✗';
507
- output.writeln(` ${mStatus} ${m.type}`);
508
- }
509
- }
510
- output.writeln();
511
- }
512
- // Baseline comparison
513
- const MAX_BASELINE_BYTES = 5 * 1024 * 1024; // 5 MB
514
- const baselinePath = join(baselinesDir, `${benchmarkId ?? 'all'}.json`);
515
- if (existsSync(baselinePath)) {
516
- const baselineStat = statSync(baselinePath);
517
- if (baselineStat.size > MAX_BASELINE_BYTES) {
518
- output.writeln(output.error(`Baseline file too large (max 5 MB)`));
519
- return { success: false, message: 'Baseline file too large' };
520
- }
521
- const baseline = JSON.parse(readFileSync(baselinePath, 'utf-8'));
522
- const hasRegression = runner.detectRegression(results, baseline);
523
- if (hasRegression) {
524
- output.writeln(output.error(`Regression detected — pass rate dropped below baseline (${(baseline.passRate * 100).toFixed(0)}%)`));
525
- }
526
- else {
527
- output.writeln(output.success('No regression detected vs baseline'));
528
- }
529
- }
530
- // Pin baseline
531
- if (pinBaseline) {
532
- if (!existsSync(baselinesDir))
533
- mkdirSync(baselinesDir, { recursive: true });
534
- const id = benchmarkId ?? 'all';
535
- const baseline = runner.pinBaseline(id, results);
536
- const baselineTmp = baselinePath + '.tmp';
537
- writeFileSync(baselineTmp, JSON.stringify(baseline, null, 2));
538
- renameSync(baselineTmp, baselinePath);
539
- output.writeln(output.success(`Baseline pinned: ${(baseline.passRate * 100).toFixed(0)}% pass rate`));
540
- }
541
- const allPassed = results.every((r) => r.passed);
542
- return { success: allPassed, message: allPassed ? 'All metrics passed' : 'Some metrics failed' };
543
- },
544
- };
545
- // ============================================================================
546
- // Main Benchmark Command
547
- // ============================================================================
548
- export const benchmarkCommand = {
549
- name: 'benchmark',
550
- description: 'Performance benchmarking for self-learning and neural systems',
551
- subcommands: [
552
- pretrainCommand,
553
- neuralCommand,
554
- memoryCommand,
555
- allCommand,
556
- regressionCommand,
557
- ],
558
- examples: [
559
- { command: 'monomind benchmark pretrain', description: 'Benchmark pre-training system' },
560
- { command: 'monomind benchmark neural', description: 'Benchmark neural operations' },
561
- { command: 'monomind benchmark memory', description: 'Benchmark memory operations' },
562
- { command: 'monomind benchmark all', description: 'Run all benchmarks' },
563
- { command: 'monomind benchmark regression', description: 'List quality regression benchmarks' },
564
- { command: 'monomind benchmark regression -b my-bench -a output.txt', description: 'Evaluate agent output against a benchmark' },
565
- ],
566
- action: async (_ctx) => {
567
- output.writeln();
568
- output.writeln(output.bold('Monomind Benchmark Suite'));
569
- output.writeln(output.dim('─'.repeat(50)));
570
- output.writeln();
571
- output.writeln('Available subcommands:');
572
- output.writeln(` ${output.highlight('pretrain')} - Benchmark pattern-learning pre-training`);
573
- output.writeln(` ${output.highlight('neural')} - Benchmark neural operations (embeddings, WASM)`);
574
- output.writeln(` ${output.highlight('memory')} - Benchmark memory operations (HNSW, store, search)`);
575
- output.writeln(` ${output.highlight('all')} - Run all benchmark suites`);
576
- output.writeln(` ${output.highlight('regression')} - Quality regression testing with baselines`);
577
- output.writeln();
578
- output.writeln('Examples:');
579
- output.writeln(' monomind benchmark pretrain -i 200');
580
- output.writeln(' monomind benchmark all --save results.json');
581
- output.writeln();
582
- return { success: true, message: 'Use a subcommand to run benchmarks' };
583
- },
584
- };
585
- export default benchmarkCommand;
586
- //# sourceMappingURL=benchmark.js.map
@@ -1,8 +0,0 @@
1
- /**
2
- * CLI Migrate Command
3
- * Migration tools for V2 to v1 transition
4
- */
5
- import type { Command } from '../types.js';
6
- export declare const migrateCommand: Command;
7
- export default migrateCommand;
8
- //# sourceMappingURL=migrate.d.ts.map