monomind 1.9.16 → 1.10.0

package/packages/@monomind/cli/dist/src/benchmarks/benchmark-runner.d.ts

@@ -1,87 +0,0 @@
-/**
- * Benchmark Runner for Regression Testing (Task 34)
- * Loads benchmark definitions, evaluates quality metrics, and detects regressions.
- */
-type BenchmarkDefinition = any;
-type BenchmarkResult = any;
-type BenchmarkBaseline = any;
-type QualityMetric = any;
-type MetricResult = any;
-export declare class BenchmarkRunner {
-    private baselines;
-    /**
-     * Load benchmark definitions from JSON files in a directory.
-     * Each JSON file should contain a single BenchmarkDefinition or an array of them.
-     */
-    loadBenchmarks(dir: string): BenchmarkDefinition[];
-    /**
-     * Evaluate quality metrics against an output string.
-     */
-    evaluateMetrics(output: string, metrics: QualityMetric[]): MetricResult[];
-    /**
-     * Run a single benchmark against provided agent output.
-     */
-    runBenchmark(def: BenchmarkDefinition, agentOutput: string): BenchmarkResult;
-    /**
-     * Pin current results as the baseline for a benchmark.
-     */
-    pinBaseline(benchmarkId: string, results: BenchmarkResult[]): BenchmarkBaseline;
-    /**
-     * Detect regression by comparing current results against a baseline.
-     * Returns true if the current pass rate is strictly below the baseline pass rate.
-     */
-    detectRegression(current: BenchmarkResult[], baseline: BenchmarkBaseline): boolean;
-    /**
-     * Get a stored baseline by benchmark ID.
-     */
-    getBaseline(benchmarkId: string): BenchmarkBaseline | undefined;
-    private evaluateSingleMetric;
-}
-/** Minimal local type aliases so SwarmBench doesn't depend on the broken
- *  @monomind/shared exports at the top of this file. */
-export interface SwarmBenchTask {
-    /** Unique task ID */
-    id: string;
-    /** One of the 5 SwarmBench coordination categories */
-    type: 'pursuit' | 'synchronization' | 'foraging' | 'flocking' | 'transport';
-    /** Human-readable description */
-    description: string;
-    /** Number of agents participating */
-    agentCount: number;
-    /** Steps / budget allowed */
-    stepBudget: number;
-    /** Success criterion function */
-    evaluate: (agentOutputs: string[]) => SwarmBenchResult;
-}
-export interface SwarmBenchResult {
-    taskId: string;
-    type: SwarmBenchTask['type'];
-    passed: boolean;
-    score: number;
-    stepsTaken: number;
-    agentCount: number;
-    details: string;
-}
-export declare const SWARM_BENCH_TASKS: readonly SwarmBenchTask[];
-/**
- * SwarmBenchRunner — runs the 5 SwarmBench coordination task types.
- * Wraps BenchmarkRunner for regression baseline tracking.
- *
- * Source: https://arxiv.org/abs/2505.04364
- */
-export declare class SwarmBenchRunner {
-    private readonly inner;
-    private baselines;
-    /** Run all (or a subset of) SwarmBench tasks against simulated agent outputs. */
-    runAll(agentOutputsByTask: Map<string, string[]>): SwarmBenchResult[];
-    /** Run a single task type by id. */
-    runTask(taskId: string, agentOutputs: string[]): SwarmBenchResult | undefined;
-    /** Pin current results as the regression baseline. */
-    pinBaseline(results: SwarmBenchResult[]): void;
-    /** Detect regression: returns task IDs whose score dropped below baseline. */
-    detectRegressions(current: SwarmBenchResult[]): string[];
-    /** Expose underlying BenchmarkRunner for general benchmarks. */
-    get benchmarkRunner(): BenchmarkRunner;
-}
-export {};
-//# sourceMappingURL=benchmark-runner.d.ts.map

package/packages/@monomind/cli/dist/src/benchmarks/benchmark-runner.js

@@ -1,273 +0,0 @@
-/**
- * Benchmark Runner for Regression Testing (Task 34)
- * Loads benchmark definitions, evaluates quality metrics, and detects regressions.
- */
-import * as fs from 'fs';
-import * as path from 'path';
-import { randomUUID } from 'crypto';
-import { containsExpected, lengthRange, noHallucination, jsonValid, customRegex, } from './metric-evaluators.js';
-export class BenchmarkRunner {
-    baselines = new Map();
-    /**
-     * Load benchmark definitions from JSON files in a directory.
-     * Each JSON file should contain a single BenchmarkDefinition or an array of them.
-     */
-    loadBenchmarks(dir) {
-        const benchmarks = [];
-        // Safe-root constraint: reject any path that escapes the working directory
-        const safeRoot = path.resolve(process.cwd());
-        const resolved = path.resolve(dir);
-        const rel = path.relative(safeRoot, resolved);
-        if (rel.startsWith('..') || path.isAbsolute(rel))
-            return [];
-        if (!fs.existsSync(resolved)) {
-            return benchmarks;
-        }
-        const files = fs.readdirSync(resolved).filter((f) => f.endsWith('.json'));
-        for (const file of files) {
-            const filePath = path.join(resolved, file);
-            const raw = fs.readFileSync(filePath, 'utf-8');
-            try {
-                const parsed = JSON.parse(raw);
-                if (Array.isArray(parsed)) {
-                    benchmarks.push(...parsed);
-                }
-                else {
-                    benchmarks.push(parsed);
-                }
-            }
-            catch {
-                // skip malformed file
-                continue;
-            }
-        }
-        return benchmarks;
-    }
-    /**
-     * Evaluate quality metrics against an output string.
-     */
-    evaluateMetrics(output, metrics) {
-        return metrics.map((metric) => this.evaluateSingleMetric(output, metric));
-    }
-    /**
-     * Run a single benchmark against provided agent output.
-     */
-    runBenchmark(def, agentOutput) {
-        const startTime = Date.now();
-        const metricResults = this.evaluateMetrics(agentOutput, def.qualityMetrics);
-        const durationMs = Date.now() - startTime;
-        const passed = metricResults.every((r) => r.passed);
-        return {
-            benchmarkId: def.benchmarkId,
-            runId: randomUUID(),
-            agentSlug: def.agentSlug,
-            passed,
-            metricResults,
-            runAt: new Date().toISOString(),
-            durationMs,
-        };
-    }
-    /**
-     * Pin current results as the baseline for a benchmark.
-     */
-    pinBaseline(benchmarkId, results) {
-        const relevantResults = results.filter((r) => r.benchmarkId === benchmarkId);
-        if (relevantResults.length === 0) {
-            const baseline = {
-                pinnedAt: new Date().toISOString(),
-                passRate: 0,
-                avgDurationMs: 0,
-            };
-            this.baselines.set(benchmarkId, baseline);
-            return baseline;
-        }
-        const passCount = relevantResults.filter((r) => r.passed).length;
-        const passRate = passCount / relevantResults.length;
-        const avgDurationMs = relevantResults.reduce((sum, r) => sum + r.durationMs, 0) /
-            relevantResults.length;
-        const baseline = {
-            pinnedAt: new Date().toISOString(),
-            passRate,
-            avgDurationMs,
-        };
-        this.baselines.set(benchmarkId, baseline);
-        return baseline;
-    }
-    /**
-     * Detect regression by comparing current results against a baseline.
-     * Returns true if the current pass rate is strictly below the baseline pass rate.
-     */
-    detectRegression(current, baseline) {
-        if (current.length === 0) {
-            return false;
-        }
-        const passCount = current.filter((r) => r.passed).length;
-        const currentPassRate = passCount / current.length;
-        return currentPassRate < baseline.passRate;
-    }
-    /**
-     * Get a stored baseline by benchmark ID.
-     */
-    getBaseline(benchmarkId) {
-        return this.baselines.get(benchmarkId);
-    }
-    // ------------------------------------------------------------------
-    // Private helpers
-    // ------------------------------------------------------------------
-    evaluateSingleMetric(output, metric) {
-        switch (metric.type) {
-            case 'contains_expected':
-                return containsExpected(output, metric.config);
-            case 'length_range':
-                return lengthRange(output, metric.config);
-            case 'no_hallucination':
-                return noHallucination(output, metric.config);
-            case 'json_valid':
-                return jsonValid(output);
-            case 'custom_regex':
-                return customRegex(output, metric.config);
-            default:
-                return {
-                    type: metric.type,
-                    passed: false,
-                    actual: null,
-                    expected: null,
-                    message: `Unknown metric type: ${metric.type}`,
-                };
-        }
-    }
-}
-// ------------------------------------------------------------------
-// Default benchmark suite (5 coordination task types)
-// ------------------------------------------------------------------
-/** Pursuit — agents must collectively report a common target string. */
-const pursuitTask = {
-    id: 'swarm-pursuit-01',
-    type: 'pursuit',
-    description: 'All agents converge on and name the same moving target within the step budget',
-    agentCount: 4,
-    stepBudget: 10,
-    evaluate(agentOutputs) {
-        const target = 'TARGET_FOUND';
-        const hits = agentOutputs.filter(o => o.includes(target)).length;
-        const score = agentOutputs.length > 0 ? hits / agentOutputs.length : 0;
-        return { taskId: this.id, type: this.type, passed: score >= 0.75, score, stepsTaken: 1, agentCount: agentOutputs.length, details: `${hits}/${agentOutputs.length} agents converged` };
-    },
-};
-/** Synchronization — all agents must emit the same token at the same logical step. */
-const synchronizationTask = {
-    id: 'swarm-sync-01',
-    type: 'synchronization',
-    description: 'Agents reach consensus on a shared value simultaneously without direct communication',
-    agentCount: 6,
-    stepBudget: 5,
-    evaluate(agentOutputs) {
-        if (agentOutputs.length === 0)
-            return { taskId: this.id, type: this.type, passed: false, score: 0, stepsTaken: 0, agentCount: 0, details: 'no outputs' };
-        const counts = new Map();
-        for (const o of agentOutputs) {
-            const tok = o.trim().split(/\s+/)[0] ?? '';
-            counts.set(tok, (counts.get(tok) ?? 0) + 1);
-        }
-        const maxAgree = Math.max(...counts.values());
-        const score = maxAgree / agentOutputs.length;
-        return { taskId: this.id, type: this.type, passed: score >= 0.8, score, stepsTaken: 1, agentCount: agentOutputs.length, details: `majority agreement: ${(score * 100).toFixed(0)}%` };
-    },
-};
-/** Foraging — agents must collectively mention ≥ N distinct resources. */
-const foragingTask = {
-    id: 'swarm-forage-01',
-    type: 'foraging',
-    description: 'Agents distribute to discover and collect distinct resources from the environment',
-    agentCount: 5,
-    stepBudget: 15,
-    evaluate(agentOutputs) {
-        const resourcePattern = /RESOURCE_[A-Z]+/g;
-        const found = new Set();
-        for (const o of agentOutputs) {
-            for (const m of o.matchAll(resourcePattern))
-                found.add(m[0]);
-        }
-        const goal = 4;
-        const score = Math.min(1, found.size / goal);
-        return { taskId: this.id, type: this.type, passed: found.size >= goal, score, stepsTaken: 1, agentCount: agentOutputs.length, details: `${found.size}/${goal} resources found: ${[...found].join(', ')}` };
-    },
-};
-/** Flocking — each agent must maintain formation by referencing its neighbour. */
-const flockingTask = {
-    id: 'swarm-flock-01',
-    type: 'flocking',
-    description: 'Agents maintain cohesion and alignment while navigating toward a goal',
-    agentCount: 8,
-    stepBudget: 20,
-    evaluate(agentOutputs) {
-        const cohesionKeyword = 'NEIGHBOUR_OK';
-        const aligned = agentOutputs.filter(o => o.includes(cohesionKeyword)).length;
-        const score = agentOutputs.length > 0 ? aligned / agentOutputs.length : 0;
-        return { taskId: this.id, type: this.type, passed: score >= 0.7, score, stepsTaken: 1, agentCount: agentOutputs.length, details: `${aligned}/${agentOutputs.length} in formation` };
-    },
-};
-/** Transport — item must appear in successive agent outputs showing relay progress. */
-const transportTask = {
-    id: 'swarm-transport-01',
-    type: 'transport',
-    description: 'Agents relay a payload across a chain; payload must appear in last agent output',
-    agentCount: 4,
-    stepBudget: 8,
-    evaluate(agentOutputs) {
-        const payload = 'PAYLOAD_DELIVERED';
-        const lastAgentDelivered = agentOutputs.length > 0 && agentOutputs[agentOutputs.length - 1].includes(payload);
-        const relayCount = agentOutputs.filter(o => o.includes('RELAY')).length;
-        const score = lastAgentDelivered ? 1 : relayCount / Math.max(agentOutputs.length, 1);
-        return { taskId: this.id, type: this.type, passed: lastAgentDelivered, score, stepsTaken: agentOutputs.length, agentCount: agentOutputs.length, details: lastAgentDelivered ? 'payload delivered' : `relay progress: ${relayCount}/${agentOutputs.length}` };
-    },
-};
-export const SWARM_BENCH_TASKS = [
-    pursuitTask,
-    synchronizationTask,
-    foragingTask,
-    flockingTask,
-    transportTask,
-];
-/**
- * SwarmBenchRunner — runs the 5 SwarmBench coordination task types.
- * Wraps BenchmarkRunner for regression baseline tracking.
- *
- * Source: https://arxiv.org/abs/2505.04364
- */
-export class SwarmBenchRunner {
-    inner = new BenchmarkRunner();
-    baselines = new Map();
-    /** Run all (or a subset of) SwarmBench tasks against simulated agent outputs. */
-    runAll(agentOutputsByTask) {
-        return SWARM_BENCH_TASKS.map(task => {
-            const outputs = agentOutputsByTask.get(task.id) ?? [];
-            return task.evaluate(outputs);
-        });
-    }
-    /** Run a single task type by id. */
-    runTask(taskId, agentOutputs) {
-        const task = SWARM_BENCH_TASKS.find(t => t.id === taskId);
-        return task?.evaluate(agentOutputs);
-    }
-    /** Pin current results as the regression baseline. */
-    pinBaseline(results) {
-        for (const r of results) {
-            this.baselines.set(r.taskId, r);
-        }
-    }
-    /** Detect regression: returns task IDs whose score dropped below baseline. */
-    detectRegressions(current) {
-        return current
-            .filter(r => {
-            const baseline = this.baselines.get(r.taskId);
-            return baseline !== undefined && r.score < baseline.score;
-        })
-            .map(r => r.taskId);
-    }
-    /** Expose underlying BenchmarkRunner for general benchmarks. */
-    get benchmarkRunner() {
-        return this.inner;
-    }
-}
-//# sourceMappingURL=benchmark-runner.js.map

package/packages/@monomind/cli/dist/src/benchmarks/metric-evaluators.d.ts

@@ -1,36 +0,0 @@
-/**
- * Metric Evaluators for Benchmark Runner (Task 34)
- * Individual metric evaluation functions for quality assessment.
- */
-type MetricResult = any;
-/**
- * Checks whether the output contains the expected substring.
- */
-export declare function containsExpected(output: string, config: {
-    expected: string;
-}): MetricResult;
-/**
- * Checks whether the output length falls within the specified range.
- */
-export declare function lengthRange(output: string, config: {
-    min: number;
-    max: number;
-}): MetricResult;
-/**
- * Checks that the output does not contain any forbidden words (hallucination markers).
- */
-export declare function noHallucination(output: string, config: {
-    forbidden: string[];
-}): MetricResult;
-/**
- * Checks whether the output is valid JSON.
- */
-export declare function jsonValid(output: string): MetricResult;
-/**
- * Checks whether the output matches a custom regex pattern.
- */
-export declare function customRegex(output: string, config: {
-    pattern: string;
-}): MetricResult;
-export {};
-//# sourceMappingURL=metric-evaluators.d.ts.map

package/packages/@monomind/cli/dist/src/benchmarks/metric-evaluators.js

@@ -1,114 +0,0 @@
-/**
- * Metric Evaluators for Benchmark Runner (Task 34)
- * Individual metric evaluation functions for quality assessment.
- */
-/**
- * Checks whether the output contains the expected substring.
- */
-export function containsExpected(output, config) {
-    const found = output.includes(config.expected);
-    return {
-        type: 'contains_expected',
-        passed: found,
-        actual: found ? config.expected : null,
-        expected: config.expected,
-        message: found
-            ? `Output contains expected string "${config.expected}"`
-            : `Output missing expected string "${config.expected}"`,
-    };
-}
-/**
- * Checks whether the output length falls within the specified range.
- */
-export function lengthRange(output, config) {
-    const len = output.length;
-    const passed = len >= config.min && len <= config.max;
-    return {
-        type: 'length_range',
-        passed,
-        actual: len,
-        expected: { min: config.min, max: config.max },
-        message: passed
-            ? `Output length ${len} within range [${config.min}, ${config.max}]`
-            : `Output length ${len} outside range [${config.min}, ${config.max}]`,
-    };
-}
-/**
- * Checks that the output does not contain any forbidden words (hallucination markers).
- */
-export function noHallucination(output, config) {
-    const lowerOutput = output.toLowerCase();
-    const found = config.forbidden.filter((word) => lowerOutput.includes(word.toLowerCase()));
-    const passed = found.length === 0;
-    return {
-        type: 'no_hallucination',
-        passed,
-        actual: found.length > 0 ? found : null,
-        expected: null,
-        message: passed
-            ? 'No forbidden words found in output'
-            : `Forbidden words found: ${found.join(', ')}`,
-    };
-}
-/**
- * Checks whether the output is valid JSON.
- */
-export function jsonValid(output) {
-    let passed = false;
-    let parsedType = null;
-    try {
-        const parsed = JSON.parse(output);
-        passed = true;
-        parsedType = typeof parsed;
-    }
-    catch {
-        // not valid JSON
-    }
-    return {
-        type: 'json_valid',
-        passed,
-        actual: passed ? parsedType : 'invalid',
-        expected: 'valid JSON',
-        message: passed ? 'Output is valid JSON' : 'Output is not valid JSON',
-    };
-}
-/**
- * Checks whether the output matches a custom regex pattern.
- */
-export function customRegex(output, config) {
-    // Reject overly long patterns and those with nested/repeated quantifiers
-    // (catastrophic backtracking — a malicious benchmark definition could
-    // pin CI runners with `^(a+)+$` against a long output string).
-    if (typeof config.pattern !== 'string' || config.pattern.length > 200) {
-        return {
-            type: 'custom_regex',
-            passed: false,
-            actual: null,
-            expected: config.pattern,
-            message: 'Pattern rejected: too long or invalid',
-        };
-    }
-    if (/(\(.*[+*?].*\)|[+*?]){2,}|\{[0-9,]+\}.*[+*?]|\([^)]*\|[^)]*\)[+*?{]/.test(config.pattern)) {
-        return {
-            type: 'custom_regex',
-            passed: false,
-            actual: null,
-            expected: config.pattern,
-            message: 'Pattern rejected: nested quantifiers risk catastrophic backtracking',
-        };
-    }
-    // Cap output length so even slow patterns can't burn unlimited CPU
-    const boundedOutput = output.length > 1024 * 1024 ? output.slice(0, 1024 * 1024) : output;
-    const regex = new RegExp(config.pattern);
-    const match = regex.test(boundedOutput);
-    return {
-        type: 'custom_regex',
-        passed: match,
-        actual: match ? boundedOutput.match(regex)?.[0] ?? null : null,
-        expected: config.pattern,
-        message: match
-            ? `Output matches pattern /${config.pattern}/`
-            : `Output does not match pattern /${config.pattern}/`,
-    };
-}
-//# sourceMappingURL=metric-evaluators.js.map

package/packages/@monomind/cli/dist/src/benchmarks/pretrain/index.d.ts

@@ -1,58 +0,0 @@
-/**
- * Self-Learning Pre-Training Benchmark Suite
- * Comprehensive benchmarks for SONA, EWC++, MoE, pattern learning
- *
- * Metrics measured:
- * - SONA adaptation latency (<0.05ms target)
- * - Pattern learning throughput
- * - EWC++ consolidation effectiveness
- * - Memory retrieval accuracy
- * - Pre-training convergence speed
- *
- * @module v1/cli/benchmarks/pretrain
- */
-export interface BenchmarkResult {
-    name: string;
-    iterations: number;
-    meanMs: number;
-    medianMs: number;
-    p95Ms: number;
-    p99Ms: number;
-    minMs: number;
-    maxMs: number;
-    stdDev: number;
-    opsPerSecond: number;
-    targetMet: boolean;
-    targetMs?: number;
-}
-export interface BenchmarkSuite {
-    name: string;
-    results: BenchmarkResult[];
-    totalDurationMs: number;
-    timestamp: string;
-    environment: {
-        nodeVersion: string;
-        platform: string;
-        arch: string;
-        cpuCount: number;
-    };
-}
-export interface BenchmarkConfig {
-    iterations: number;
-    warmupIterations: number;
-    targetMs?: number;
-    verbose?: boolean;
-}
-export declare function formatBenchmarkResult(result: BenchmarkResult): string;
-export declare function runBenchmark(name: string, fn: () => Promise<void> | void, config: BenchmarkConfig): Promise<BenchmarkResult>;
-export declare function benchmarkSONAAdaptation(config: BenchmarkConfig): Promise<BenchmarkResult>;
-export declare function benchmarkPatternLearning(config: BenchmarkConfig): Promise<BenchmarkResult>;
-export declare function benchmarkEWCConsolidation(config: BenchmarkConfig): Promise<BenchmarkResult>;
-export declare function benchmarkMemoryRetrieval(config: BenchmarkConfig): Promise<BenchmarkResult>;
-export declare function benchmarkEmbeddingGeneration(config: BenchmarkConfig): Promise<BenchmarkResult>;
-export declare function benchmarkMoERouting(config: BenchmarkConfig): Promise<BenchmarkResult>;
-export declare function benchmarkBatchCosine(config: BenchmarkConfig): Promise<BenchmarkResult>;
-export declare function benchmarkPretrainPipeline(config: BenchmarkConfig): Promise<BenchmarkResult>;
-export declare function runPretrainBenchmarkSuite(config?: Partial<BenchmarkConfig>): Promise<BenchmarkSuite>;
-export default runPretrainBenchmarkSuite;
-//# sourceMappingURL=index.d.ts.map