@monoes/cli 1.0.0

package/dist/src/autopilot-state.js

@@ -0,0 +1,279 @@
+/**
+ * Autopilot Shared State Module
+ *
+ * Centralizes state management, validation, and task discovery
+ * for both CLI command and MCP tools. Eliminates code duplication.
+ *
+ * ADR-072: Autopilot Integration
+ * Security: Addresses prototype pollution, NaN bypass, input validation
+ */
+// ── Constants ─────────────────────────────────────────────────
+export const STATE_DIR = '.monobrain/data';
+export const STATE_FILE = `${STATE_DIR}/autopilot-state.json`;
+export const LOG_FILE = `${STATE_DIR}/autopilot-log.json`;
+/** Maximum entries kept in state.history (prevents unbounded growth) */
+const MAX_HISTORY_ENTRIES = 50;
+/** Maximum entries kept in the event log */
+const MAX_LOG_ENTRIES = 1000;
+/** Allowlist for valid task sources */
+export const VALID_TASK_SOURCES = new Set(['team-tasks', 'swarm-tasks', 'file-checklist']);
+/** Terminal task statuses */
+export const TERMINAL_STATUSES = new Set(['completed', 'done', 'cancelled', 'skipped', 'failed']);
+// ── Validation Helpers ────────────────────────────────────────
+/**
+ * Sanitize a parsed JSON object to prevent prototype pollution.
+ * Removes __proto__, constructor, and prototype keys recursively.
+ */
+function sanitizeObject(obj) {
+    if (obj === null || typeof obj !== 'object')
+        return obj;
+    if (Array.isArray(obj))
+        return obj.map(sanitizeObject);
+    const clean = {};
+    for (const key of Object.keys(obj)) {
+        if (key === '__proto__' || key === 'constructor' || key === 'prototype')
+            continue;
+        clean[key] = sanitizeObject(obj[key]);
+    }
+    return clean;
+}
+/**
+ * Safe JSON.parse that prevents prototype pollution.
+ */
+export function safeJsonParse(raw) {
+    return sanitizeObject(JSON.parse(raw));
+}
+/**
+ * Validate and coerce a numeric parameter. Returns the default if
+ * the input is NaN, undefined, or outside the allowed range.
+ */
+export function validateNumber(value, min, max, defaultValue) {
+    if (value === undefined || value === null)
+        return defaultValue;
+    const num = Number(value);
+    if (!Number.isFinite(num))
+        return defaultValue;
+    return Math.min(Math.max(min, Math.round(num)), max);
+}
+/**
+ * Validate task sources against the allowlist.
+ * Returns only valid sources; falls back to defaults if none are valid.
+ */
+export function validateTaskSources(sources) {
+    const defaults = ['team-tasks', 'swarm-tasks', 'file-checklist'];
+    if (!Array.isArray(sources))
+        return defaults;
+    const valid = sources
+        .filter((s) => typeof s === 'string')
+        .map(s => s.trim())
+        .filter(s => VALID_TASK_SOURCES.has(s));
+    return valid.length > 0 ? valid : defaults;
+}
+// ── State Management ──────────────────────────────────────────
+export function getDefaultState() {
+    const crypto = require('crypto');
+    return {
+        sessionId: crypto.randomUUID(),
+        enabled: false,
+        startTime: Date.now(),
+        iterations: 0,
+        maxIterations: 50,
+        timeoutMinutes: 240,
+        taskSources: ['team-tasks', 'swarm-tasks', 'file-checklist'],
+        lastCheck: null,
+        history: [],
+    };
+}
+export function loadState() {
+    const fs = require('fs');
+    const path = require('path');
+    const filePath = path.resolve(STATE_FILE);
+    const defaults = getDefaultState();
+    try {
+        if (fs.existsSync(filePath)) {
+            const raw = safeJsonParse(fs.readFileSync(filePath, 'utf-8'));
+            const merged = { ...defaults, ...raw };
+            // Re-validate fields that could be tampered with
+            merged.maxIterations = validateNumber(merged.maxIterations, 1, 1000, 50);
+            merged.timeoutMinutes = validateNumber(merged.timeoutMinutes, 1, 1440, 240);
+            merged.iterations = validateNumber(merged.iterations, 0, 1000, 0);
+            merged.taskSources = validateTaskSources(merged.taskSources);
+            // Cap history to prevent unbounded growth
+            if (Array.isArray(merged.history) && merged.history.length > MAX_HISTORY_ENTRIES) {
+                merged.history = merged.history.slice(-MAX_HISTORY_ENTRIES);
+            }
+            return merged;
+        }
+    }
+    catch {
+        // Corrupted state file — return defaults
+    }
+    return defaults;
+}
+export function saveState(state) {
+    const fs = require('fs');
+    const path = require('path');
+    const dir = path.resolve(STATE_DIR);
+    if (!fs.existsSync(dir))
+        fs.mkdirSync(dir, { recursive: true });
+    // Cap history before saving
+    if (state.history.length > MAX_HISTORY_ENTRIES) {
+        state.history = state.history.slice(-MAX_HISTORY_ENTRIES);
+    }
+    const tmpFile = path.resolve(STATE_FILE) + '.tmp';
+    fs.writeFileSync(tmpFile, JSON.stringify(state, null, 2));
+    fs.renameSync(tmpFile, path.resolve(STATE_FILE));
+}
+export function appendLog(entry) {
+    const fs = require('fs');
+    const path = require('path');
+    const filePath = path.resolve(LOG_FILE);
+    const dir = path.resolve(STATE_DIR);
+    if (!fs.existsSync(dir))
+        fs.mkdirSync(dir, { recursive: true });
+    let log = [];
+    try {
+        if (fs.existsSync(filePath)) {
+            log = safeJsonParse(fs.readFileSync(filePath, 'utf-8'));
+            if (!Array.isArray(log))
+                log = [];
+        }
+    }
+    catch {
+        log = [];
+    }
+    log.push(entry);
+    if (log.length > MAX_LOG_ENTRIES)
+        log = log.slice(-MAX_LOG_ENTRIES);
+    const tmpFile = filePath + '.tmp';
+    fs.writeFileSync(tmpFile, JSON.stringify(log, null, 2));
+    fs.renameSync(tmpFile, filePath);
+}
+export function loadLog() {
+    const fs = require('fs');
+    const path = require('path');
+    const filePath = path.resolve(LOG_FILE);
+    try {
+        if (fs.existsSync(filePath)) {
+            const result = safeJsonParse(fs.readFileSync(filePath, 'utf-8'));
+            return Array.isArray(result) ? result : [];
+        }
+    }
+    catch {
+        // Corrupted log — return empty
+    }
+    return [];
+}
+// ── Task Discovery ────────────────────────────────────────────
+export function discoverTasks(sources) {
+    const fs = require('fs');
+    const path = require('path');
+    const os = require('os');
+    const tasks = [];
+    // Only process valid sources
+    const validSources = sources.filter(s => VALID_TASK_SOURCES.has(s));
+    for (const source of validSources) {
+        if (source === 'team-tasks') {
+            const tasksDir = path.join(os.homedir(), '.claude', 'tasks');
+            try {
+                if (fs.existsSync(tasksDir)) {
+                    const teams = fs.readdirSync(tasksDir, { withFileTypes: true });
+                    for (const team of teams) {
+                        if (!team.isDirectory())
+                            continue;
+                        const teamDir = path.join(tasksDir, team.name);
+                        const files = fs.readdirSync(teamDir).filter((f) => f.endsWith('.json'));
+                        for (const file of files) {
+                            try {
+                                const data = safeJsonParse(fs.readFileSync(path.join(teamDir, file), 'utf-8'));
+                                tasks.push({
+                                    id: String(data.id || file.replace('.json', '')),
+                                    subject: String(data.subject || data.title || file),
+                                    status: String(data.status || 'unknown'),
+                                    source: 'team-tasks',
+                                });
+                            }
+                            catch { /* skip individual file */ }
+                        }
+                    }
+                }
+            }
+            catch { /* skip source */ }
+        }
+        if (source === 'swarm-tasks') {
+            const swarmFile = path.resolve('.monobrain/swarm-tasks.json');
+            try {
+                if (fs.existsSync(swarmFile)) {
+                    const data = safeJsonParse(fs.readFileSync(swarmFile, 'utf-8'));
+                    const swarmTasks = Array.isArray(data) ? data : (data.tasks || []);
+                    for (const t of swarmTasks) {
+                        if (t && typeof t === 'object') {
+                            const task = t;
+                            tasks.push({
+                                id: String(task.id || task.taskId || `swarm-${tasks.length}`),
+                                subject: String(task.subject || task.description || task.name || 'Unnamed task'),
+                                status: String(task.status || 'unknown'),
+                                source: 'swarm-tasks',
+                            });
+                        }
+                    }
+                }
+            }
+            catch { /* skip source */ }
+        }
+        if (source === 'file-checklist') {
+            const checklistFile = path.resolve('.monobrain/data/checklist.json');
+            try {
+                if (fs.existsSync(checklistFile)) {
+                    const data = safeJsonParse(fs.readFileSync(checklistFile, 'utf-8'));
+                    const items = Array.isArray(data) ? data : (data.items || []);
+                    for (const item of items) {
+                        if (item && typeof item === 'object') {
+                            const i = item;
+                            tasks.push({
+                                id: String(i.id || `check-${tasks.length}`),
+                                subject: String(i.subject || i.text || i.description || 'Unnamed item'),
+                                status: String(i.status || (i.done ? 'completed' : 'pending')),
+                                source: 'file-checklist',
+                            });
+                        }
+                    }
+                }
+            }
+            catch { /* skip source */ }
+        }
+    }
+    return tasks;
+}
+// ── Progress Helpers ──────────────────────────────────────────
+export function isTerminal(status) {
+    return TERMINAL_STATUSES.has(status.toLowerCase());
+}
+export function getProgress(tasks) {
+    const completed = tasks.filter(t => isTerminal(t.status)).length;
+    const total = tasks.length;
+    const percent = total === 0 ? 100 : Math.round((completed / total) * 100);
+    const incomplete = tasks.filter(t => !isTerminal(t.status));
+    return { completed, total, percent, incomplete };
+}
+// ── Reward Calculation ────────────────────────────────────────
+export function calculateReward(iterations, durationMs) {
+    const iterFactor = (1 - iterations / (iterations + 10)) * 0.6;
+    const timeFactor = (1 - Math.min(durationMs / 3600000, 1)) * 0.4;
+    return Math.round((iterFactor + timeFactor) * 100) / 100;
+}
+// ── Learning Integration ──────────────────────────────────────
+export async function tryLoadLearning() {
+    try {
+        const modPath = 'agentic-flow/dist/coordination/autopilot-learning.js';
+        const mod = await import(/* webpackIgnore: true */ modPath).catch(() => null);
+        if (mod?.AutopilotLearning) {
+            const instance = new mod.AutopilotLearning();
+            if (await instance.initialize())
+                return instance;
+        }
+    }
+    catch { /* not available */ }
+    return null;
+}
+//# sourceMappingURL=autopilot-state.js.map

package/dist/src/autopilot-state.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"autopilot-state.js","sourceRoot":"","sources":["../../src/autopilot-state.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,iEAAiE;AAEjE,MAAM,CAAC,MAAM,SAAS,GAAG,iBAAiB,CAAC;AAC3C,MAAM,CAAC,MAAM,UAAU,GAAG,GAAG,SAAS,uBAAuB,CAAC;AAC9D,MAAM,CAAC,MAAM,QAAQ,GAAG,GAAG,SAAS,qBAAqB,CAAC;AAE1D,wEAAwE;AACxE,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAE/B,4CAA4C;AAC5C,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B,uCAAuC;AACvC,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,YAAY,EAAE,aAAa,EAAE,gBAAgB,CAAC,CAAC,CAAC;AAE3F,6BAA6B;AAC7B,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;AAoClG,iEAAiE;AAEjE;;;GAGG;AACH,SAAS,cAAc,CAAC,GAAY;IAClC,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,GAAG,CAAC;IACxD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAEvD,MAAM,KAAK,GAA4B,EAAE,CAAC;IAC1C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAA8B,CAAC,EAAE,CAAC;QAC9D,IAAI,GAAG,KAAK,WAAW,IAAI,GAAG,KAAK,aAAa,IAAI,GAAG,KAAK,WAAW;YAAE,SAAS;QAClF,KAAK,CAAC,GAAG,CAAC,GAAG,cAAc,CAAE,GAA+B,CAAC,GAAG,CAAC,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAI,GAAW;IAC1C,OAAO,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAM,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,KAAc,EAAE,GAAW,EAAE,GAAW,EAAE,YAAoB;IAC3F,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,YAAY,CAAC;IAC/D,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,YAAY,CAAC;IAC/C,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACvD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAgB;IAClD,MAAM,QAAQ,GAAG,CAAC,YAAY,EAAE,aAAa,EAAE,gBAAgB,CAAC,CAAC;IACjE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC7C,MAAM,KAAK,GAAG,OAAO;SAClB,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;SACjD,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SAClB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1C,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC7C,CAAC;AAED,iEAAiE;AAEjE,MAAM,UAAU,eAAe;IAC7B,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAA4B,CAAC;IAC5D,OAAO;QACL,SAAS,EAAE,MAAM,CAAC,UAAU,EAAE;QAC9B,OAAO,EAAE,KAAK;QACd,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QACrB,UAAU,EAAE,CAAC;QACb,aAAa,EAAE,EAAE;QACjB,cAAc,EAAE,GAAG;QACnB,WAAW,EAAE,CAAC,YAAY,EAAE,aAAa,EAAE,gBAAgB,CAAC;QAC5D,SAAS,EAAE,IAAI;QACf,OAAO,EAAE,EAAE;KACZ,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAwB,CAAC;IAChD,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAA0B,CAAC;IACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAC;IACnC,IAAI,CAAC;QACH,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,aAAa,CAA0B,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;YACvF,MAAM,MAAM,GAAG,EAAE,GAAG,QAAQ,EAAE,GAAG,GAAG,EAAE,CAAC;YACvC,iDAAiD;YACjD,MAAM,CAAC,aAAa,GAAG,cAAc,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;YACzE,MAAM,CAAC,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;YAC5E,MAAM,CAAC,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAClE,MAAM,CAAC,WAAW,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YAC7D,0CAA0C;YAC1C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,mBAAmB,EAAE,CAAC;gBACjF,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,mBAAmB,CAAC,CAAC;YAC9D,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,yCAAyC;IAC3C,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAAqB;IAC7C,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAwB,CAAC;IAChD,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAA0B,CAAC;IACtD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACpC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChE,4BAA4B;IAC5B,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,mBAAmB,EAAE,CAAC;QAC/C,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,mBAAmB,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC;IAClD,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1D,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAAwB;IAChD,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAwB,CAAC;IAChD,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAA0B,CAAC;IACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACpC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChE,IAAI,GAAG,GAAwB,EAAE,CAAC;IAClC,IAAI,CAAC;QACH,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC5B,GAAG,GAAG,aAAa,CAAsB,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;YAC7E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;gBAAE,GAAG,GAAG,EAAE,CAAC;QACpC,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,GAAG,GAAG,EAAE,CAAC;IACX,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChB,IAAI,GAAG,CAAC,MAAM,GAAG,eAAe;QAAE,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,CAAC;IACpE,MAAM,OAAO,GAAG,QAAQ,GAAG,MAAM,CAAC;IAClC,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACxD,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,OAAO;IACrB,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAwB,CAAC;IAChD,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAA0B,CAAC;IACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,CAAC;QACH,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,aAAa,CAAsB,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;YACtF,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,+BAA+B;IACjC,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,iEAAiE;AAEjE,MAAM,UAAU,aAAa,CAAC,OAAiB;IAC7C,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAwB,CAAC;IAChD,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAA0B,CAAC;IACtD,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAwB,CAAC;IAChD,MAAM,KAAK,GAAe,EAAE,CAAC;IAE7B,6BAA6B;IAC7B,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAEpE,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;QAClC,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YAC7D,IAAI,CAAC;gBACH,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC5B,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;oBAChE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;wBACzB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;4BAAE,SAAS;wBAClC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;wBAC/C,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;wBACjF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;4BACzB,IAAI,CAAC;gCACH,MAAM,IAAI,GAAG,aAAa,CAA0B,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;gCACxG,KAAK,CAAC,IAAI,CAAC;oCACT,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;oCAChD,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC;oCACnD,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,SAAS,CAAC;oCACxC,MAAM,EAAE,YAAY;iCACrB,CAAC,CAAC;4BACL,CAAC;4BAAC,MAAM,CAAC,CAAC,0BAA0B,CAAC,CAAC;wBACxC,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;QAC/B,CAAC;QAED,IAAI,MAAM,KAAK,aAAa,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,6BAA6B,CAAC,CAAC;YAC9D,IAAI,CAAC;gBACH,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC7B,MAAM,IAAI,GAAG,aAAa,CAAsC,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;oBACrG,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAE,IAAgC,CAAC,KAAkB,IAAI,EAAE,CAAC,CAAC;oBAC7G,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;wBAC3B,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;4BAC/B,MAAM,IAAI,GAAG,CAA4B,CAAC;4BAC1C,KAAK,CAAC,IAAI,CAAC;gCACT,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,MAAM,IAAI,SAAS,KAAK,CAAC,MAAM,EAAE,CAAC;gCAC7D,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,IAAI,IAAI,cAAc,CAAC;gCAChF,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,SAAS,CAAC;gCACxC,MAAM,EAAE,aAAa;6BACtB,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;QAC/B,CAAC;QAED,IAAI,MAAM,KAAK,gBAAgB,EAAE,CAAC;YAChC,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC;YACrE,IAAI,CAAC;gBACH,IAAI,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;oBACjC,MAAM,IAAI,GAAG,aAAa,CAAsC,EAAE,CAAC,YAAY,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC,CAAC;oBACzG,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAE,IAAgC,CAAC,KAAkB,IAAI,EAAE,CAAC,CAAC;oBACxG,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;wBACzB,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;4BACrC,MAAM,CAAC,GAAG,IAA+B,CAAC;4BAC1C,KAAK,CAAC,IAAI,CAAC;gCACT,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,SAAS,KAAK,CAAC,MAAM,EAAE,CAAC;gCAC3C,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,WAAW,IAAI,cAAc,CAAC;gCACvE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;gCAC9D,MAAM,EAAE,gBAAgB;6BACzB,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,iEAAiE;AAEjE,MAAM,UAAU,UAAU,CAAC,MAAc;IACvC,OAAO,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;AACrD,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAiB;IAC3C,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IACjE,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC3B,MAAM,OAAO,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;IAC1E,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5D,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;AACnD,CAAC;AAED,iEAAiE;AAEjE,MAAM,UAAU,eAAe,CAAC,UAAkB,EAAE,UAAkB;IACpE,MAAM,UAAU,GAAG,CAAC,CAAC,GAAG,UAAU,GAAG,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC;IAC9D,MAAM,UAAU,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;IACjE,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,UAAU,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;AAC3D,CAAC;AAED,iEAAiE;AAEjE,MAAM,CAAC,KAAK,UAAU,eAAe;IACnC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,sDAAsD,CAAC;QACvE,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAC9E,IAAI,GAAG,EAAE,iBAAiB,EAAE,CAAC;YAC3B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,iBAAiB,EAAE,CAAC;YAC7C,IAAI,MAAM,QAAQ,CAAC,UAAU,EAAE;gBAAE,OAAO,QAAQ,CAAC;QACnD,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,mBAAmB,CAAC,CAAC;IAC/B,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/src/benchmarks/benchmark-runner.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Benchmark Runner for Regression Testing (Task 34)
+ * Loads benchmark definitions, evaluates quality metrics, and detects regressions.
+ */
+import type { BenchmarkDefinition, BenchmarkResult, BenchmarkBaseline, QualityMetric, MetricResult } from '@monobrain/shared';
+export declare class BenchmarkRunner {
+    private baselines;
+    /**
+     * Load benchmark definitions from JSON files in a directory.
+     * Each JSON file should contain a single BenchmarkDefinition or an array of them.
+     */
+    loadBenchmarks(dir: string): BenchmarkDefinition[];
+    /**
+     * Evaluate quality metrics against an output string.
+     */
+    evaluateMetrics(output: string, metrics: QualityMetric[]): MetricResult[];
+    /**
+     * Run a single benchmark against provided agent output.
+     */
+    runBenchmark(def: BenchmarkDefinition, agentOutput: string): BenchmarkResult;
+    /**
+     * Pin current results as the baseline for a benchmark.
+     */
+    pinBaseline(benchmarkId: string, results: BenchmarkResult[]): BenchmarkBaseline;
+    /**
+     * Detect regression by comparing current results against a baseline.
+     * Returns true if the current pass rate is strictly below the baseline pass rate.
+     */
+    detectRegression(current: BenchmarkResult[], baseline: BenchmarkBaseline): boolean;
+    /**
+     * Get a stored baseline by benchmark ID.
+     */
+    getBaseline(benchmarkId: string): BenchmarkBaseline | undefined;
+    private evaluateSingleMetric;
+}
+//# sourceMappingURL=benchmark-runner.d.ts.map

package/dist/src/benchmarks/benchmark-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark-runner.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/benchmark-runner.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,KAAK,EACV,mBAAmB,EACnB,eAAe,EACf,iBAAiB,EACjB,aAAa,EACb,YAAY,EACb,MAAM,mBAAmB,CAAC;AAS3B,qBAAa,eAAe;IAC1B,OAAO,CAAC,SAAS,CAA6C;IAE9D;;;OAGG;IACH,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,mBAAmB,EAAE;IAwBlD;;OAEG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,YAAY,EAAE;IAIzE;;OAEG;IACH,YAAY,CACV,GAAG,EAAE,mBAAmB,EACxB,WAAW,EAAE,MAAM,GAClB,eAAe;IAoBlB;;OAEG;IACH,WAAW,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,iBAAiB;IA+B/E;;;OAGG;IACH,gBAAgB,CACd,OAAO,EAAE,eAAe,EAAE,EAC1B,QAAQ,EAAE,iBAAiB,GAC1B,OAAO;IAWV;;OAEG;IACH,WAAW,CAAC,WAAW,EAAE,MAAM,GAAG,iBAAiB,GAAG,SAAS;IAQ/D,OAAO,CAAC,oBAAoB;CA+B7B"}

package/dist/src/benchmarks/benchmark-runner.js

@@ -0,0 +1,128 @@
+/**
+ * Benchmark Runner for Regression Testing (Task 34)
+ * Loads benchmark definitions, evaluates quality metrics, and detects regressions.
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import { randomUUID } from 'crypto';
+import { containsExpected, lengthRange, noHallucination, jsonValid, customRegex, } from './metric-evaluators.js';
+export class BenchmarkRunner {
+    baselines = new Map();
+    /**
+     * Load benchmark definitions from JSON files in a directory.
+     * Each JSON file should contain a single BenchmarkDefinition or an array of them.
+     */
+    loadBenchmarks(dir) {
+        const benchmarks = [];
+        if (!fs.existsSync(dir)) {
+            return benchmarks;
+        }
+        const files = fs.readdirSync(dir).filter((f) => f.endsWith('.json'));
+        for (const file of files) {
+            const filePath = path.join(dir, file);
+            const raw = fs.readFileSync(filePath, 'utf-8');
+            const parsed = JSON.parse(raw);
+            if (Array.isArray(parsed)) {
+                benchmarks.push(...parsed);
+            }
+            else {
+                benchmarks.push(parsed);
+            }
+        }
+        return benchmarks;
+    }
+    /**
+     * Evaluate quality metrics against an output string.
+     */
+    evaluateMetrics(output, metrics) {
+        return metrics.map((metric) => this.evaluateSingleMetric(output, metric));
+    }
+    /**
+     * Run a single benchmark against provided agent output.
+     */
+    runBenchmark(def, agentOutput) {
+        const startTime = Date.now();
+        const metricResults = this.evaluateMetrics(agentOutput, def.qualityMetrics);
+        const durationMs = Date.now() - startTime;
+        const passed = metricResults.every((r) => r.passed);
+        return {
+            benchmarkId: def.benchmarkId,
+            runId: randomUUID(),
+            agentSlug: def.agentSlug,
+            passed,
+            metricResults,
+            runAt: new Date().toISOString(),
+            durationMs,
+        };
+    }
+    /**
+     * Pin current results as the baseline for a benchmark.
+     */
+    pinBaseline(benchmarkId, results) {
+        const relevantResults = results.filter((r) => r.benchmarkId === benchmarkId);
+        if (relevantResults.length === 0) {
+            const baseline = {
+                pinnedAt: new Date().toISOString(),
+                passRate: 0,
+                avgDurationMs: 0,
+            };
+            this.baselines.set(benchmarkId, baseline);
+            return baseline;
+        }
+        const passCount = relevantResults.filter((r) => r.passed).length;
+        const passRate = passCount / relevantResults.length;
+        const avgDurationMs = relevantResults.reduce((sum, r) => sum + r.durationMs, 0) /
+            relevantResults.length;
+        const baseline = {
+            pinnedAt: new Date().toISOString(),
+            passRate,
+            avgDurationMs,
+        };
+        this.baselines.set(benchmarkId, baseline);
+        return baseline;
+    }
+    /**
+     * Detect regression by comparing current results against a baseline.
+     * Returns true if the current pass rate is strictly below the baseline pass rate.
+     */
+    detectRegression(current, baseline) {
+        if (current.length === 0) {
+            return false;
+        }
+        const passCount = current.filter((r) => r.passed).length;
+        const currentPassRate = passCount / current.length;
+        return currentPassRate < baseline.passRate;
+    }
+    /**
+     * Get a stored baseline by benchmark ID.
+     */
+    getBaseline(benchmarkId) {
+        return this.baselines.get(benchmarkId);
+    }
+    // ------------------------------------------------------------------
+    // Private helpers
+    // ------------------------------------------------------------------
+    evaluateSingleMetric(output, metric) {
+        switch (metric.type) {
+            case 'contains_expected':
+                return containsExpected(output, metric.config);
+            case 'length_range':
+                return lengthRange(output, metric.config);
+            case 'no_hallucination':
+                return noHallucination(output, metric.config);
+            case 'json_valid':
+                return jsonValid(output);
+            case 'custom_regex':
+                return customRegex(output, metric.config);
+            default:
+                return {
+                    type: metric.type,
+                    passed: false,
+                    actual: null,
+                    expected: null,
+                    message: `Unknown metric type: ${metric.type}`,
+                };
+        }
+    }
+}
+//# sourceMappingURL=benchmark-runner.js.map

package/dist/src/benchmarks/benchmark-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark-runner.js","sourceRoot":"","sources":["../../../src/benchmarks/benchmark-runner.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAQpC,OAAO,EACL,gBAAgB,EAChB,WAAW,EACX,eAAe,EACf,SAAS,EACT,WAAW,GACZ,MAAM,wBAAwB,CAAC;AAEhC,MAAM,OAAO,eAAe;IAClB,SAAS,GAAmC,IAAI,GAAG,EAAE,CAAC;IAE9D;;;OAGG;IACH,cAAc,CAAC,GAAW;QACxB,MAAM,UAAU,GAA0B,EAAE,CAAC;QAE7C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,OAAO,UAAU,CAAC;QACpB,CAAC;QAED,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAErE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YACtC,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAE/B,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC1B,UAAU,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,MAAc,EAAE,OAAwB;QACtD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED;;OAEG;IACH,YAAY,CACV,GAAwB,EACxB,WAAmB;QAEnB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,aAAa,GAAG,IAAI,CAAC,eAAe,CACxC,WAAW,EACX,GAAG,CAAC,cAAc,CACnB,CAAC;QACF,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAC1C,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAEpD,OAAO;YACL,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,KAAK,EAAE,UAAU,EAAE;YACnB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,MAAM;YACN,aAAa;YACb,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC/B,UAAU;SACX,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,WAAmB,EAAE,OAA0B;QACzD,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,CACpC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,WAAW,CACrC,CAAC;QAEF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAsB;gBAClC,QAAQ,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBAClC,QAAQ,EAAE,CAAC;gBACX,aAAa,EAAE,CAAC;aACjB,CAAC;YACF,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;YAC1C,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QACjE,MAAM,QAAQ,GAAG,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC;QACpD,MAAM,aAAa,GACjB,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;YACzD,eAAe,CAAC,MAAM,CAAC;QAEzB,MAAM,QAAQ,GAAsB;YAClC,QAAQ,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAClC,QAAQ;YACR,aAAa;SACd,CAAC;QAEF,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;QAC1C,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;OAGG;IACH,gBAAgB,CACd,OAA0B,EAC1B,QAA2B;QAE3B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QACzD,MAAM,eAAe,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC;QAEnD,OAAO,eAAe,GAAG,QAAQ,CAAC,QAAQ,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,WAAmB;QAC7B,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IACzC,CAAC;IAED,qEAAqE;IACrE,kBAAkB;IAClB,qEAAqE;IAE7D,oBAAoB,CAC1B,MAAc,EACd,MAAqB;QAErB,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;YACpB,KAAK,mBAAmB;gBACtB,OAAO,gBAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,MAA8B,CAAC,CAAC;YACzE,KAAK,cAAc;gBACjB,OAAO,WAAW,CAChB,MAAM,EACN,MAAM,CAAC,MAAsC,CAC9C,CAAC;YACJ,KAAK,kBAAkB;gBACrB,OAAO,eAAe,CACpB,MAAM,EACN,MAAM,CAAC,MAAiC,CACzC,CAAC;YACJ,KAAK,YAAY;gBACf,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC;YAC3B,KAAK,cAAc;gBACjB,OAAO,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,MAA6B,CAAC,CAAC;YACnE;gBACE,OAAO;oBACL,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,MAAM,EAAE,KAAK;oBACb,MAAM,EAAE,IAAI;oBACZ,QAAQ,EAAE,IAAI;oBACd,OAAO,EAAE,wBAAwB,MAAM,CAAC,IAAI,EAAE;iBAC/C,CAAC;QACN,CAAC;IACH,CAAC;CACF"}

package/dist/src/benchmarks/metric-evaluators.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Metric Evaluators for Benchmark Runner (Task 34)
+ * Individual metric evaluation functions for quality assessment.
+ */
+import type { MetricResult } from '@monobrain/shared';
+/**
+ * Checks whether the output contains the expected substring.
+ */
+export declare function containsExpected(output: string, config: {
+    expected: string;
+}): MetricResult;
+/**
+ * Checks whether the output length falls within the specified range.
+ */
+export declare function lengthRange(output: string, config: {
+    min: number;
+    max: number;
+}): MetricResult;
+/**
+ * Checks that the output does not contain any forbidden words (hallucination markers).
+ */
+export declare function noHallucination(output: string, config: {
+    forbidden: string[];
+}): MetricResult;
+/**
+ * Checks whether the output is valid JSON.
+ */
+export declare function jsonValid(output: string): MetricResult;
+/**
+ * Checks whether the output matches a custom regex pattern.
+ */
+export declare function customRegex(output: string, config: {
+    pattern: string;
+}): MetricResult;
+//# sourceMappingURL=metric-evaluators.d.ts.map

package/dist/src/benchmarks/metric-evaluators.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metric-evaluators.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/metric-evaluators.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEtD;;GAEG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,GAC3B,YAAY,CAWd;AAED;;GAEG;AACH,wBAAgB,WAAW,CACzB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GACnC,YAAY,CAYd;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE;IAAE,SAAS,EAAE,MAAM,EAAE,CAAA;CAAE,GAC9B,YAAY,CAed;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAiBtD;AAED;;GAEG;AACH,wBAAgB,WAAW,CACzB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,GAC1B,YAAY,CAYd"}

package/dist/src/benchmarks/metric-evaluators.js

@@ -0,0 +1,91 @@
+/**
+ * Metric Evaluators for Benchmark Runner (Task 34)
+ * Individual metric evaluation functions for quality assessment.
+ */
+/**
+ * Checks whether the output contains the expected substring.
+ */
+export function containsExpected(output, config) {
+    const found = output.includes(config.expected);
+    return {
+        type: 'contains_expected',
+        passed: found,
+        actual: found ? config.expected : null,
+        expected: config.expected,
+        message: found
+            ? `Output contains expected string "${config.expected}"`
+            : `Output missing expected string "${config.expected}"`,
+    };
+}
+/**
+ * Checks whether the output length falls within the specified range.
+ */
+export function lengthRange(output, config) {
+    const len = output.length;
+    const passed = len >= config.min && len <= config.max;
+    return {
+        type: 'length_range',
+        passed,
+        actual: len,
+        expected: { min: config.min, max: config.max },
+        message: passed
+            ? `Output length ${len} within range [${config.min}, ${config.max}]`
+            : `Output length ${len} outside range [${config.min}, ${config.max}]`,
+    };
+}
+/**
+ * Checks that the output does not contain any forbidden words (hallucination markers).
+ */
+export function noHallucination(output, config) {
+    const lowerOutput = output.toLowerCase();
+    const found = config.forbidden.filter((word) => lowerOutput.includes(word.toLowerCase()));
+    const passed = found.length === 0;
+    return {
+        type: 'no_hallucination',
+        passed,
+        actual: found.length > 0 ? found : null,
+        expected: null,
+        message: passed
+            ? 'No forbidden words found in output'
+            : `Forbidden words found: ${found.join(', ')}`,
+    };
+}
+/**
+ * Checks whether the output is valid JSON.
+ */
+export function jsonValid(output) {
+    let passed = false;
+    let parsedType = null;
+    try {
+        const parsed = JSON.parse(output);
+        passed = true;
+        parsedType = typeof parsed;
+    }
+    catch {
+        // not valid JSON
+    }
+    return {
+        type: 'json_valid',
+        passed,
+        actual: passed ? parsedType : 'invalid',
+        expected: 'valid JSON',
+        message: passed ? 'Output is valid JSON' : 'Output is not valid JSON',
+    };
+}
+/**
+ * Checks whether the output matches a custom regex pattern.
+ */
+export function customRegex(output, config) {
+    const regex = new RegExp(config.pattern);
+    const match = regex.test(output);
+    return {
+        type: 'custom_regex',
+        passed: match,
+        actual: match ? output.match(regex)?.[0] ?? null : null,
+        expected: config.pattern,
+        message: match
+            ? `Output matches pattern /${config.pattern}/`
+            : `Output does not match pattern /${config.pattern}/`,
+    };
+}
+//# sourceMappingURL=metric-evaluators.js.map

package/dist/src/benchmarks/metric-evaluators.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metric-evaluators.js","sourceRoot":"","sources":["../../../src/benchmarks/metric-evaluators.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAC9B,MAAc,EACd,MAA4B;IAE5B,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO;QACL,IAAI,EAAE,mBAAmB;QACzB,MAAM,EAAE,KAAK;QACb,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;QACtC,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,OAAO,EAAE,KAAK;YACZ,CAAC,CAAC,oCAAoC,MAAM,CAAC,QAAQ,GAAG;YACxD,CAAC,CAAC,mCAAmC,MAAM,CAAC,QAAQ,GAAG;KAC1D,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CACzB,MAAc,EACd,MAAoC;IAEpC,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC;IAC1B,MAAM,MAAM,GAAG,GAAG,IAAI,MAAM,CAAC,GAAG,IAAI,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC;IACtD,OAAO;QACL,IAAI,EAAE,cAAc;QACpB,MAAM;QACN,MAAM,EAAE,GAAG;QACX,QAAQ,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE;QAC9C,OAAO,EAAE,MAAM;YACb,CAAC,CAAC,iBAAiB,GAAG,kBAAkB,MAAM,CAAC,GAAG,KAAK,MAAM,CAAC,GAAG,GAAG;YACpE,CAAC,CAAC,iBAAiB,GAAG,mBAAmB,MAAM,CAAC,GAAG,KAAK,MAAM,CAAC,GAAG,GAAG;KACxE,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAC7B,MAAc,EACd,MAA+B;IAE/B,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAC7C,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CACzC,CAAC;IACF,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC;IAClC,OAAO;QACL,IAAI,EAAE,kBAAkB;QACxB,MAAM;QACN,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;QACvC,QAAQ,EAAE,IAAI;QACd,OAAO,EAAE,MAAM;YACb,CAAC,CAAC,oCAAoC;YACtC,CAAC,CAAC,0BAA0B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KACjD,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,MAAc;IACtC,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,IAAI,UAAU,GAAkB,IAAI,CAAC;IACrC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAClC,MAAM,GAAG,IAAI,CAAC;QACd,UAAU,GAAG,OAAO,MAAM,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,iBAAiB;IACnB,CAAC;IACD,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,MAAM;QACN,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;QACvC,QAAQ,EAAE,YAAY;QACtB,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,0BAA0B;KACtE,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CACzB,MAAc,EACd,MAA2B;IAE3B,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACjC,OAAO;QACL,IAAI,EAAE,cAAc;QACpB,MAAM,EAAE,KAAK;QACb,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI;QACvD,QAAQ,EAAE,MAAM,CAAC,OAAO;QACxB,OAAO,EAAE,KAAK;YACZ,CAAC,CAAC,2BAA2B,MAAM,CAAC,OAAO,GAAG;YAC9C,CAAC,CAAC,kCAAkC,MAAM,CAAC,OAAO,GAAG;KACxD,CAAC;AACJ,CAAC"}