monomind 1.18.9 → 1.18.10

package/packages/@monomind/cli/dist/src/benchmarks/benchmark-runner.js

@@ -1,340 +0,0 @@
-/**
- * Benchmark Runner for Regression Testing (Task 34)
- * Loads benchmark definitions, evaluates quality metrics, and detects regressions.
- */
-import * as fs from 'fs';
-import * as path from 'path';
-import { randomUUID } from 'crypto';
-import { containsExpected, lengthRange, noHallucination, jsonValid, customRegex, } from './metric-evaluators.js';
-export class BenchmarkRunner {
-    baselines = new Map();
-    /**
-     * Load benchmark definitions from JSON files in a directory.
-     * Each JSON file should contain a single BenchmarkDefinition or an array of them.
-     */
-    loadBenchmarks(dir) {
-        const benchmarks = [];
-        // Safe-root constraint: reject any path that escapes the working directory
-        const safeRoot = path.resolve(process.cwd());
-        const resolved = path.resolve(dir);
-        const rel = path.relative(safeRoot, resolved);
-        if (rel.startsWith('..') || path.isAbsolute(rel))
-            return [];
-        if (!fs.existsSync(resolved)) {
-            return benchmarks;
-        }
-        const MAX_BENCH_FILE_SIZE = 1 * 1024 * 1024; // 1 MB per benchmark file
-        const files = fs.readdirSync(resolved).filter((f) => f.endsWith('.json'));
-        for (const file of files) {
-            const filePath = path.join(resolved, file);
-            // Guard against OOM: reject symlinks and oversized files before reading
-            let stat;
-            try {
-                stat = fs.lstatSync(filePath);
-            }
-            catch {
-                continue;
-            }
-            if (stat.isSymbolicLink())
-                continue;
-            if (stat.size > MAX_BENCH_FILE_SIZE)
-                continue;
-            const raw = fs.readFileSync(filePath, 'utf-8');
-            try {
-                const parsed = JSON.parse(raw);
-                if (Array.isArray(parsed)) {
-                    benchmarks.push(...parsed);
-                }
-                else {
-                    benchmarks.push(parsed);
-                }
-            }
-            catch {
-                // skip malformed file
-                continue;
-            }
-        }
-        return benchmarks;
-    }
-    /**
-     * Evaluate quality metrics against an output string.
-     */
-    evaluateMetrics(output, metrics) {
-        return metrics.map((metric) => this.evaluateSingleMetric(output, metric));
-    }
-    /**
-     * Run a single benchmark against provided agent output.
-     */
-    runBenchmark(def, agentOutput) {
-        const startTime = Date.now();
-        const metricResults = this.evaluateMetrics(agentOutput, def.qualityMetrics);
-        const durationMs = Date.now() - startTime;
-        const passed = metricResults.every((r) => r.passed);
-        return {
-            benchmarkId: def.benchmarkId,
-            runId: randomUUID(),
-            agentSlug: def.agentSlug,
-            passed,
-            metricResults,
-            runAt: new Date().toISOString(),
-            durationMs,
-        };
-    }
-    /**
-     * Pin current results as the baseline for a benchmark.
-     */
-    pinBaseline(benchmarkId, results) {
-        const relevantResults = results.filter((r) => r.benchmarkId === benchmarkId);
-        if (relevantResults.length === 0) {
-            const baseline = {
-                pinnedAt: new Date().toISOString(),
-                passRate: 0,
-                avgDurationMs: 0,
-            };
-            this.baselines.set(benchmarkId, baseline);
-            return baseline;
-        }
-        const passCount = relevantResults.filter((r) => r.passed).length;
-        const passRate = passCount / relevantResults.length;
-        const avgDurationMs = relevantResults.reduce((sum, r) => sum + r.durationMs, 0) /
-            relevantResults.length;
-        const baseline = {
-            pinnedAt: new Date().toISOString(),
-            passRate,
-            avgDurationMs,
-        };
-        this.baselines.set(benchmarkId, baseline);
-        return baseline;
-    }
-    /**
-     * Detect regression by comparing current results against a baseline.
-     * Returns true if the current pass rate is strictly below the baseline pass rate.
-     */
-    detectRegression(current, baseline) {
-        if (current.length === 0) {
-            return false;
-        }
-        const passCount = current.filter((r) => r.passed).length;
-        const currentPassRate = passCount / current.length;
-        return currentPassRate < baseline.passRate;
-    }
-    /**
-     * Get a stored baseline by benchmark ID.
-     */
-    getBaseline(benchmarkId) {
-        return this.baselines.get(benchmarkId);
-    }
-    // ------------------------------------------------------------------
-    // Private helpers
-    // ------------------------------------------------------------------
-    evaluateSingleMetric(output, metric) {
-        switch (metric.type) {
-            case 'contains_expected':
-                return containsExpected(output, metric.config);
-            case 'length_range':
-                return lengthRange(output, metric.config);
-            case 'no_hallucination':
-                return noHallucination(output, metric.config);
-            case 'json_valid':
-                return jsonValid(output);
-            case 'custom_regex':
-                return customRegex(output, metric.config);
-            default:
-                return {
-                    type: metric.type,
-                    passed: false,
-                    actual: null,
-                    expected: null,
-                    message: `Unknown metric type: ${metric.type}`,
-                };
-        }
-    }
-}
-// ------------------------------------------------------------------
-// Default benchmark suite (5 coordination task types)
-// ------------------------------------------------------------------
-/** Pursuit — agents must collectively report a common target string. */
-const pursuitTask = {
-    id: 'swarm-pursuit-01',
-    type: 'pursuit',
-    description: 'All agents converge on and name the same moving target within the step budget',
-    agentCount: 4,
-    stepBudget: 10,
-    evaluate(agentOutputs) {
-        const target = 'TARGET_FOUND';
-        const hits = agentOutputs.filter((o) => o.includes(target)).length;
-        const score = agentOutputs.length > 0 ? hits / agentOutputs.length : 0;
-        return {
-            taskId: this.id,
-            type: this.type,
-            passed: score >= 0.75,
-            score,
-            stepsTaken: 1,
-            agentCount: agentOutputs.length,
-            details: `${hits}/${agentOutputs.length} agents converged`,
-        };
-    },
-};
-/** Synchronization — all agents must emit the same token at the same logical step. */
-const synchronizationTask = {
-    id: 'swarm-sync-01',
-    type: 'synchronization',
-    description: 'Agents reach consensus on a shared value simultaneously without direct communication',
-    agentCount: 6,
-    stepBudget: 5,
-    evaluate(agentOutputs) {
-        if (agentOutputs.length === 0)
-            return {
-                taskId: this.id,
-                type: this.type,
-                passed: false,
-                score: 0,
-                stepsTaken: 0,
-                agentCount: 0,
-                details: 'no outputs',
-            };
-        const counts = new Map();
-        for (const o of agentOutputs) {
-            const tok = o.trim().split(/\s+/)[0] ?? '';
-            counts.set(tok, (counts.get(tok) ?? 0) + 1);
-        }
-        const maxAgree = Math.max(...counts.values());
-        const score = maxAgree / agentOutputs.length;
-        return {
-            taskId: this.id,
-            type: this.type,
-            passed: score >= 0.8,
-            score,
-            stepsTaken: 1,
-            agentCount: agentOutputs.length,
-            details: `majority agreement: ${(score * 100).toFixed(0)}%`,
-        };
-    },
-};
-/** Foraging — agents must collectively mention ≥ N distinct resources. */
-const foragingTask = {
-    id: 'swarm-forage-01',
-    type: 'foraging',
-    description: 'Agents distribute to discover and collect distinct resources from the environment',
-    agentCount: 5,
-    stepBudget: 15,
-    evaluate(agentOutputs) {
-        const resourcePattern = /RESOURCE_[A-Z]+/g;
-        const found = new Set();
-        for (const o of agentOutputs) {
-            for (const m of o.matchAll(resourcePattern))
-                found.add(m[0]);
-        }
-        const goal = 4;
-        const score = Math.min(1, found.size / goal);
-        return {
-            taskId: this.id,
-            type: this.type,
-            passed: found.size >= goal,
-            score,
-            stepsTaken: 1,
-            agentCount: agentOutputs.length,
-            details: `${found.size}/${goal} resources found: ${[...found].join(', ')}`,
-        };
-    },
-};
-/** Flocking — each agent must maintain formation by referencing its neighbour. */
-const flockingTask = {
-    id: 'swarm-flock-01',
-    type: 'flocking',
-    description: 'Agents maintain cohesion and alignment while navigating toward a goal',
-    agentCount: 8,
-    stepBudget: 20,
-    evaluate(agentOutputs) {
-        const cohesionKeyword = 'NEIGHBOUR_OK';
-        const aligned = agentOutputs.filter((o) => o.includes(cohesionKeyword)).length;
-        const score = agentOutputs.length > 0 ? aligned / agentOutputs.length : 0;
-        return {
-            taskId: this.id,
-            type: this.type,
-            passed: score >= 0.7,
-            score,
-            stepsTaken: 1,
-            agentCount: agentOutputs.length,
-            details: `${aligned}/${agentOutputs.length} in formation`,
-        };
-    },
-};
-/** Transport — item must appear in successive agent outputs showing relay progress. */
-const transportTask = {
-    id: 'swarm-transport-01',
-    type: 'transport',
-    description: 'Agents relay a payload across a chain; payload must appear in last agent output',
-    agentCount: 4,
-    stepBudget: 8,
-    evaluate(agentOutputs) {
-        const payload = 'PAYLOAD_DELIVERED';
-        const lastAgentDelivered = agentOutputs.length > 0 && agentOutputs[agentOutputs.length - 1].includes(payload);
-        const relayCount = agentOutputs.filter((o) => o.includes('RELAY')).length;
-        const score = lastAgentDelivered ? 1 : relayCount / Math.max(agentOutputs.length, 1);
-        return {
-            taskId: this.id,
-            type: this.type,
-            passed: lastAgentDelivered,
-            score,
-            stepsTaken: agentOutputs.length,
-            agentCount: agentOutputs.length,
-            details: lastAgentDelivered
-                ? 'payload delivered'
-                : `relay progress: ${relayCount}/${agentOutputs.length}`,
-        };
-    },
-};
-export const SWARM_BENCH_TASKS = [
-    pursuitTask,
-    synchronizationTask,
-    foragingTask,
-    flockingTask,
-    transportTask,
-];
-/**
- * Alias for {@link SWARM_BENCH_TASKS}. Kept for ergonomic imports.
- */
-export const SWARM = SWARM_BENCH_TASKS;
-/**
- * SwarmBenchRunner — runs the 5 SwarmBench coordination task types.
- * Wraps BenchmarkRunner for regression baseline tracking.
- *
- * Source: https://arxiv.org/abs/2505.04364
- */
-export class SwarmBenchRunner {
-    inner = new BenchmarkRunner();
-    baselines = new Map();
-    /** Run all (or a subset of) SwarmBench tasks against simulated agent outputs. */
-    runAll(agentOutputsByTask) {
-        return SWARM_BENCH_TASKS.map((task) => {
-            const outputs = agentOutputsByTask.get(task.id) ?? [];
-            return task.evaluate(outputs);
-        });
-    }
-    /** Run a single task type by id. */
-    runTask(taskId, agentOutputs) {
-        const task = SWARM_BENCH_TASKS.find((t) => t.id === taskId);
-        return task?.evaluate(agentOutputs);
-    }
-    /** Pin current results as the regression baseline. */
-    pinBaseline(results) {
-        for (const r of results) {
-            this.baselines.set(r.taskId, r);
-        }
-    }
-    /** Detect regression: returns task IDs whose score dropped below baseline. */
-    detectRegressions(current) {
-        return current
-            .filter((r) => {
-            const baseline = this.baselines.get(r.taskId);
-            return baseline !== undefined && r.score < baseline.score;
-        })
-            .map((r) => r.taskId);
-    }
-    /** Expose underlying BenchmarkRunner for general benchmarks. */
-    get benchmarkRunner() {
-        return this.inner;
-    }
-}
-//# sourceMappingURL=benchmark-runner.js.map

package/packages/@monomind/cli/dist/src/benchmarks/metric-evaluators.d.ts

@@ -1,36 +0,0 @@
-/**
- * Metric Evaluators for Benchmark Runner (Task 34)
- * Individual metric evaluation functions for quality assessment.
- */
-type MetricResult = any;
-/**
- * Checks whether the output contains the expected substring.
- */
-export declare function containsExpected(output: string, config: {
-    expected: string;
-}): MetricResult;
-/**
- * Checks whether the output length falls within the specified range.
- */
-export declare function lengthRange(output: string, config: {
-    min: number;
-    max: number;
-}): MetricResult;
-/**
- * Checks that the output does not contain any forbidden words (hallucination markers).
- */
-export declare function noHallucination(output: string, config: {
-    forbidden: string[];
-}): MetricResult;
-/**
- * Checks whether the output is valid JSON.
- */
-export declare function jsonValid(output: string): MetricResult;
-/**
- * Checks whether the output matches a custom regex pattern.
- */
-export declare function customRegex(output: string, config: {
-    pattern: string;
-}): MetricResult;
-export {};
-//# sourceMappingURL=metric-evaluators.d.ts.map

package/packages/@monomind/cli/dist/src/benchmarks/metric-evaluators.js

@@ -1,125 +0,0 @@
-/**
- * Metric Evaluators for Benchmark Runner (Task 34)
- * Individual metric evaluation functions for quality assessment.
- */
-/**
- * Checks whether the output contains the expected substring.
- */
-export function containsExpected(output, config) {
-    // Cap expected to prevent huge strings from inflating returned objects
-    const expected = typeof config.expected === 'string' ? config.expected.slice(0, 200) : '';
-    const found = output.includes(expected);
-    return {
-        type: 'contains_expected',
-        passed: found,
-        actual: found ? expected : null,
-        expected,
-        message: found
-            ? `Output contains expected string "${expected}"`
-            : `Output missing expected string "${expected}"`,
-    };
-}
-/**
- * Checks whether the output length falls within the specified range.
- */
-export function lengthRange(output, config) {
-    const len = output.length;
-    const passed = len >= config.min && len <= config.max;
-    return {
-        type: 'length_range',
-        passed,
-        actual: len,
-        expected: { min: config.min, max: config.max },
-        message: passed
-            ? `Output length ${len} within range [${config.min}, ${config.max}]`
-            : `Output length ${len} outside range [${config.min}, ${config.max}]`,
-    };
-}
-/**
- * Checks that the output does not contain any forbidden words (hallucination markers).
- */
-export function noHallucination(output, config) {
-    const lowerOutput = output.toLowerCase();
-    // Cap forbidden array to 200 entries to prevent unbounded O(n) scan
-    const forbidden = Array.isArray(config.forbidden) ? config.forbidden.slice(0, 200) : [];
-    const found = forbidden.filter((word) => typeof word === 'string' && lowerOutput.includes(word.toLowerCase()));
-    const passed = found.length === 0;
-    // Truncate reflected words to avoid inflated messages
-    const truncatedFound = found.map((w) => w.slice(0, 50));
-    return {
-        type: 'no_hallucination',
-        passed,
-        actual: found.length > 0 ? truncatedFound : null,
-        expected: null,
-        message: passed
-            ? 'No forbidden words found in output'
-            : `Forbidden words found: ${truncatedFound.join(', ')}`,
-    };
-}
-/**
- * Checks whether the output is valid JSON.
- */
-export function jsonValid(output) {
-    let passed = false;
-    let parsedType = null;
-    // Cap output before JSON.parse to prevent OOM on huge strings
-    const MAX_JSON_BYTES = 1 * 1024 * 1024; // 1 MB
-    const bounded = typeof output === 'string' && output.length > MAX_JSON_BYTES
-        ? output.slice(0, MAX_JSON_BYTES)
-        : output;
-    try {
-        const parsed = JSON.parse(bounded);
-        passed = true;
-        parsedType = typeof parsed;
-    }
-    catch {
-        // not valid JSON
-    }
-    return {
-        type: 'json_valid',
-        passed,
-        actual: passed ? parsedType : 'invalid',
-        expected: 'valid JSON',
-        message: passed ? 'Output is valid JSON' : 'Output is not valid JSON',
-    };
-}
-/**
- * Checks whether the output matches a custom regex pattern.
- */
-export function customRegex(output, config) {
-    // Reject overly long patterns and those with nested/repeated quantifiers
-    // (catastrophic backtracking — a malicious benchmark definition could
-    // pin CI runners with `^(a+)+$` against a long output string).
-    if (typeof config.pattern !== 'string' || config.pattern.length > 200) {
-        return {
-            type: 'custom_regex',
-            passed: false,
-            actual: null,
-            expected: config.pattern,
-            message: 'Pattern rejected: too long or invalid',
-        };
-    }
-    if (/(\(.*[+*?].*\)|[+*?]){2,}|\{[0-9,]+\}.*[+*?]|\([^)]*\|[^)]*\)[+*?{]/.test(config.pattern)) {
-        return {
-            type: 'custom_regex',
-            passed: false,
-            actual: null,
-            expected: config.pattern,
-            message: 'Pattern rejected: nested quantifiers risk catastrophic backtracking',
-        };
-    }
-    // Cap output length so even slow patterns can't burn unlimited CPU
-    const boundedOutput = output.length > 1024 * 1024 ? output.slice(0, 1024 * 1024) : output;
-    const regex = new RegExp(config.pattern);
-    const match = regex.test(boundedOutput);
-    return {
-        type: 'custom_regex',
-        passed: match,
-        actual: match ? boundedOutput.match(regex)?.[0] ?? null : null,
-        expected: config.pattern,
-        message: match
-            ? `Output matches pattern /${config.pattern}/`
-            : `Output does not match pattern /${config.pattern}/`,
-    };
-}
-//# sourceMappingURL=metric-evaluators.js.map

package/packages/@monomind/cli/dist/src/commands/benchmark.d.ts

@@ -1,10 +0,0 @@
-/**
- * CLI Benchmark Command
- * Comprehensive benchmarking for self-learning, pre-training, and neural systems
- *
- * @module v1/cli/commands/benchmark
- */
-import type { Command } from '../types.js';
-export declare const benchmarkCommand: Command;
-export default benchmarkCommand;
-//# sourceMappingURL=benchmark.d.ts.map