magi-ai 0.1.0

package/dist/src/benchmark/reporter.js

@@ -0,0 +1,107 @@
+/**
+ * Generate a Markdown benchmark report comparing single-Claude vs MAGI 3-body results.
+ */
+export function generateReport(singleResult, magiResult) {
+    const lines = [];
+    const now = new Date().toISOString();
+    lines.push('# MAGI Benchmark Results');
+    lines.push('');
+    lines.push('> **Note:** These results are from dry-run mode using mock adapters. They validate the scoring framework and golden task definitions, not actual AI model performance. Run `npm run benchmark` with real CLIs configured to generate production results.');
+    lines.push('');
+    lines.push(`> Generated: ${now}`);
+    lines.push('');
+    // ── Summary Table ────────────────────────────────────────────
+    lines.push('## Summary');
+    lines.push('');
+    lines.push('| Metric | Single Claude | MAGI 3-Body | Delta |');
+    lines.push('|--------|:------------:|:-----------:|:-----:|');
+    if (singleResult && magiResult) {
+        const s = singleResult.aggregate;
+        const m = magiResult.aggregate;
+        lines.push(formatRow('Weighted Score', s.weightedScore, m.weightedScore, true));
+        lines.push(formatRow('Vote Accuracy', s.voteAccuracy, m.voteAccuracy, true));
+        lines.push(formatRow('Detection Coverage', s.detectionCoverage, m.detectionCoverage, true));
+        lines.push(formatRow('Avg Duration (ms)', s.avgDurationMs, m.avgDurationMs, false));
+    }
+    else {
+        const r = singleResult ?? magiResult;
+        if (r) {
+            const a = r.aggregate;
+            const label = r.mode === 'single-claude' ? 'Single Claude' : 'MAGI 3-Body';
+            lines.push(`| Weighted Score | ${r.mode === 'single-claude' ? pct(a.weightedScore) : '-'} | ${r.mode === 'magi-3-body' ? pct(a.weightedScore) : '-'} | - |`);
+            lines.push(`| Vote Accuracy | ${r.mode === 'single-claude' ? pct(a.voteAccuracy) : '-'} | ${r.mode === 'magi-3-body' ? pct(a.voteAccuracy) : '-'} | - |`);
+            lines.push(`| Detection Coverage | ${r.mode === 'single-claude' ? pct(a.detectionCoverage) : '-'} | ${r.mode === 'magi-3-body' ? pct(a.detectionCoverage) : '-'} | - |`);
+            lines.push(`| Avg Duration (ms) | ${r.mode === 'single-claude' ? ms(a.avgDurationMs) : '-'} | ${r.mode === 'magi-3-body' ? ms(a.avgDurationMs) : '-'} | - |`);
+            void label;
+        }
+    }
+    lines.push('');
+    // ── Per-Task Results ─────────────────────────────────────────
+    const results = [singleResult, magiResult].filter(Boolean);
+    for (const result of results) {
+        const modeLabel = result.mode === 'single-claude' ? 'Single Claude' : 'MAGI 3-Body';
+        lines.push(`## ${modeLabel} — Task Details`);
+        lines.push('');
+        lines.push(`- **Run ID**: ${result.runId}`);
+        lines.push(`- **Git Hash**: ${result.gitHash}`);
+        lines.push(`- **Timestamp**: ${result.timestamp}`);
+        lines.push(`- **Total Duration**: ${ms(result.aggregate.totalDurationMs)}`);
+        lines.push('');
+        lines.push('| Task ID | Category | Vote | Coverage | FP Rate | Score | Duration |');
+        lines.push('|---------|----------|:----:|:--------:|:-------:|:-----:|:--------:|');
+        for (const task of result.tasks) {
+            const voteIcon = task.voteAccuracy === 1.0 ? 'PASS' : 'FAIL';
+            lines.push(`| ${task.taskId} | ${task.category} | ${voteIcon} | ${pct(task.detectionCoverage)} | ${pct(task.falsePositiveRate)} | ${pct(task.compositeScore)} | ${ms(task.durationMs)} |`);
+        }
+        lines.push('');
+    }
+    // ── Category Breakdown ───────────────────────────────────────
+    for (const result of results) {
+        const modeLabel = result.mode === 'single-claude' ? 'Single Claude' : 'MAGI 3-Body';
+        lines.push(`## ${modeLabel} — Category Breakdown`);
+        lines.push('');
+        const categories = new Map();
+        for (const task of result.tasks) {
+            const list = categories.get(task.category) ?? [];
+            list.push(task);
+            categories.set(task.category, list);
+        }
+        lines.push('| Category | Tasks | Avg Score | Vote Accuracy |');
+        lines.push('|----------|:-----:|:---------:|:-------------:|');
+        for (const [category, tasks] of categories) {
+            const avgScore = tasks.reduce((a, t) => a + t.compositeScore, 0) / tasks.length;
+            const avgVote = tasks.reduce((a, t) => a + t.voteAccuracy, 0) / tasks.length;
+            lines.push(`| ${category} | ${tasks.length} | ${pct(avgScore)} | ${pct(avgVote)} |`);
+        }
+        lines.push('');
+    }
+    return lines.join('\n');
+}
+// ── Formatting helpers ────────────────────────────────────────
+function pct(value) {
+    return `${(value * 100).toFixed(1)}%`;
+}
+function ms(value) {
+    return `${Math.round(value)}ms`;
+}
+function formatRow(label, singleVal, magiVal, higherIsBetter) {
+    const isScore = label.includes('Score') || label.includes('Accuracy') || label.includes('Coverage');
+    const singleStr = isScore ? pct(singleVal) : ms(singleVal);
+    const magiStr = isScore ? pct(magiVal) : ms(magiVal);
+    const diff = magiVal - singleVal;
+    let deltaStr;
+    if (isScore) {
+        const sign = diff >= 0 ? '+' : '';
+        deltaStr = `${sign}${(diff * 100).toFixed(1)}pp`;
+    }
+    else {
+        const sign = diff >= 0 ? '+' : '';
+        deltaStr = `${sign}${Math.round(diff)}ms`;
+    }
+    // Highlight improvement vs degradation
+    const improved = higherIsBetter ? diff > 0 : diff < 0;
+    if (diff !== 0) {
+        deltaStr = improved ? `**${deltaStr}**` : deltaStr;
+    }
+    return `| ${label} | ${singleStr} | ${magiStr} | ${deltaStr} |`;
+}

package/dist/src/benchmark/runner.d.ts

@@ -0,0 +1,30 @@
+import type { BenchmarkResult, BenchmarkOptions } from './types.js';
+export declare class BenchmarkRunner {
+    private readonly options;
+    private readonly tasks;
+    constructor(options?: BenchmarkOptions);
+    /**
+     * Run a single-Claude benchmark (MELCHIOR only, quorum=1, initial-opinion only).
+     */
+    runSingle(): Promise<BenchmarkResult>;
+    /**
+     * Run a MAGI 3-body benchmark (full 3-unit deliberation).
+     */
+    runMagi(): Promise<BenchmarkResult>;
+    /**
+     * Run dry-run benchmark using MockAdapters with perfect responses.
+     */
+    private runDry;
+    /**
+     * Build a fake MagiDeliberation for dry-run scoring validation.
+     */
+    private buildMockDeliberation;
+    /**
+     * Run all benchmarks and generate report.
+     */
+    runAll(): Promise<{
+        single: BenchmarkResult | null;
+        magi: BenchmarkResult | null;
+        report: string;
+    }>;
+}

package/dist/src/benchmark/runner.js

@@ -0,0 +1,224 @@
+import { randomUUID } from 'node:crypto';
+import { execFileSync } from 'node:child_process';
+import { join } from 'node:path';
+import { safeMkdir, safeWriteFile } from '../utils/safe-fs.js';
+import { GOLDEN_TASKS } from './golden-tasks.js';
+import { scoreTask, aggregateScores } from './scorer.js';
+import { generateReport } from './reporter.js';
+import { Magi } from '../index.js';
+import { AdapterRegistry } from '../adapters/registry.js';
+import { ClaudeAdapter } from '../adapters/claude.js';
+import { PipelineRegistry } from '../pipelines/registry.js';
+import { CodeReviewPipeline } from '../pipelines/code-review.js';
+import { ArchitecturePipeline } from '../pipelines/architecture.js';
+import { BugAnalysisPipeline } from '../pipelines/bug-analysis.js';
+import { CustomPipeline } from '../pipelines/custom.js';
+import { Orchestrator } from '../engine/orchestrator.js';
+import { MockContextManager } from '../../test/helpers/mock-context-manager.js';
+import { logger } from '../utils/logger.js';
+function getGitHash() {
+    try {
+        return execFileSync('git', ['rev-parse', '--short', 'HEAD'], {
+            encoding: 'utf-8',
+            timeout: 5000,
+        }).trim();
+    }
+    catch (err) {
+        logger.debug('Benchmark: git hash retrieval failed', { error: String(err) });
+        return 'unknown';
+    }
+}
+function buildPipelineRegistry() {
+    const pipelines = new PipelineRegistry();
+    pipelines.register(new CodeReviewPipeline());
+    pipelines.register(new ArchitecturePipeline());
+    pipelines.register(new BugAnalysisPipeline());
+    pipelines.register(new CustomPipeline());
+    return pipelines;
+}
+/**
+ * Create a mock response for a golden task (for dry-run mode).
+ * Returns a "perfect" response that matches the expected outcome.
+ */
+function buildMockResponse(task) {
+    return {
+        vote: task.expectation.expectedVote,
+        confidence: 0.85,
+        reasoning: `Mock analysis: ${task.expectation.rationale}`,
+        keyPoints: task.expectation.mustDetectPoints.map(String),
+    };
+}
+export class BenchmarkRunner {
+    options;
+    tasks;
+    constructor(options = {}) {
+        this.options = options;
+        if (options.category) {
+            this.tasks = GOLDEN_TASKS.filter(t => t.category === options.category);
+        }
+        else {
+            this.tasks = GOLDEN_TASKS;
+        }
+    }
+    /**
+     * Run a single-Claude benchmark (MELCHIOR only, quorum=1, initial-opinion only).
+     */
+    async runSingle() {
+        const runId = randomUUID();
+        const scores = [];
+        if (this.options.dryRun) {
+            return this.runDry('single-claude', runId);
+        }
+        // Single-Claude: Only MELCHIOR adapter, quorum=1, single phase
+        const adapters = new AdapterRegistry();
+        adapters.register(new ClaudeAdapter({ timeoutMs: 120_000, maxRetries: 2 }));
+        const pipelines = buildPipelineRegistry();
+        const context = new MockContextManager();
+        const orchestrator = new Orchestrator({
+            adapters,
+            pipelines,
+            contextManager: context,
+        });
+        for (const goldenTask of this.tasks) {
+            const task = {
+                ...goldenTask.task,
+                config: {
+                    phases: ['initial-opinion'],
+                    consensus: { quorum: 1 },
+                },
+            };
+            const deliberation = await orchestrator.deliberate(task);
+            scores.push(scoreTask(goldenTask, deliberation));
+        }
+        return {
+            runId,
+            timestamp: new Date().toISOString(),
+            gitHash: getGitHash(),
+            mode: 'single-claude',
+            tasks: scores,
+            aggregate: aggregateScores(scores),
+        };
+    }
+    /**
+     * Run a MAGI 3-body benchmark (full 3-unit deliberation).
+     */
+    async runMagi() {
+        const runId = randomUUID();
+        const scores = [];
+        if (this.options.dryRun) {
+            return this.runDry('magi-3-body', runId);
+        }
+        const magi = new Magi({ cacheEnabled: false });
+        for (const goldenTask of this.tasks) {
+            const deliberation = await magi.deliberate(goldenTask.task);
+            scores.push(scoreTask(goldenTask, deliberation));
+        }
+        return {
+            runId,
+            timestamp: new Date().toISOString(),
+            gitHash: getGitHash(),
+            mode: 'magi-3-body',
+            tasks: scores,
+            aggregate: aggregateScores(scores),
+        };
+    }
+    /**
+     * Run dry-run benchmark using MockAdapters with perfect responses.
+     */
+    async runDry(mode, runId) {
+        const scores = [];
+        for (const goldenTask of this.tasks) {
+            const mockResp = buildMockResponse(goldenTask);
+            const deliberation = this.buildMockDeliberation(goldenTask, mockResp, mode);
+            scores.push(scoreTask(goldenTask, deliberation));
+        }
+        return {
+            runId,
+            timestamp: new Date().toISOString(),
+            gitHash: getGitHash(),
+            mode,
+            tasks: scores,
+            aggregate: aggregateScores(scores),
+        };
+    }
+    /**
+     * Build a fake MagiDeliberation for dry-run scoring validation.
+     */
+    buildMockDeliberation(goldenTask, mockResp, mode) {
+        const units = mode === 'single-claude'
+            ? ['MELCHIOR']
+            : ['MELCHIOR', 'BALTHASAR', 'CASPER'];
+        const opinions = units.map(unit => ({
+            unit,
+            vote: mockResp.vote,
+            confidence: mockResp.confidence,
+            reasoning: mockResp.reasoning,
+            keyPoints: mockResp.keyPoints,
+            rawOutput: JSON.stringify(mockResp),
+            meta: {
+                durationMs: 50,
+                exitCode: 0,
+                retryCount: 0,
+                structuredOutput: true,
+            },
+        }));
+        const isApprove = mockResp.vote === 'APPROVE';
+        const decision = mode === 'single-claude'
+            ? (isApprove ? 'UNANIMOUS_APPROVE' : 'UNANIMOUS_REJECT')
+            : (isApprove ? 'UNANIMOUS_APPROVE' : 'UNANIMOUS_REJECT');
+        const now = new Date();
+        return {
+            id: randomUUID(),
+            task: goldenTask.task,
+            rounds: [{
+                    roundNumber: 1,
+                    phase: 'initial-opinion',
+                    opinions,
+                    startedAt: now,
+                    completedAt: now,
+                }],
+            consensus: {
+                decision,
+                method: 'unanimous',
+                votes: {
+                    approve: isApprove ? units : [],
+                    reject: isApprove ? [] : units,
+                    abstain: [],
+                    weightedApprove: isApprove ? units.length : 0,
+                    weightedReject: isApprove ? 0 : units.length,
+                },
+                confidence: mockResp.confidence,
+                summary: `Mock consensus: ${decision}`,
+            },
+            totalDurationMs: 50 * units.length,
+            startedAt: now,
+            completedAt: now,
+        };
+    }
+    /**
+     * Run all benchmarks and generate report.
+     */
+    async runAll() {
+        let single = null;
+        let magi = null;
+        if (!this.options.magiOnly) {
+            single = await this.runSingle();
+        }
+        if (!this.options.singleOnly) {
+            magi = await this.runMagi();
+        }
+        const report = generateReport(single, magi);
+        // Save results
+        const outputDir = this.options.outputDir ?? 'test/benchmark/results';
+        await safeMkdir(outputDir);
+        const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+        if (single) {
+            await safeWriteFile(join(outputDir, `single-${timestamp}.json`), JSON.stringify(single, null, 2));
+        }
+        if (magi) {
+            await safeWriteFile(join(outputDir, `magi-${timestamp}.json`), JSON.stringify(magi, null, 2));
+        }
+        await safeWriteFile('docs/BENCHMARK_RESULTS.md', report);
+        return { single, magi, report };
+    }
+}

package/dist/src/benchmark/scorer.d.ts

@@ -0,0 +1,12 @@
+import type { MagiDeliberation } from '../types/core.js';
+import type { GoldenTask, TaskScore, AggregateScore } from './types.js';
+/**
+ * Score a single golden task against its deliberation result.
+ *
+ * compositeScore = voteAccuracy * 0.4 + detectionCoverage * 0.5 + (1 - falsePositiveRate) * 0.1
+ */
+export declare function scoreTask(goldenTask: GoldenTask, deliberation: MagiDeliberation): TaskScore;
+/**
+ * Aggregate scores across all tasks.
+ */
+export declare function aggregateScores(scores: readonly TaskScore[]): AggregateScore;

package/dist/src/benchmark/scorer.js

@@ -0,0 +1,124 @@
+import { getMajorityVote } from '../types/consensus.js';
+/**
+ * Extract the effective vote from a ConsensusDecision.
+ */
+function decisionToVote(decision) {
+    return getMajorityVote(decision);
+}
+/**
+ * Compute detection coverage: fraction of mustDetectPoints found in the output.
+ */
+function computeDetectionCoverage(mustDetectPoints, searchText) {
+    if (mustDetectPoints.length === 0)
+        return 1.0;
+    const lower = searchText.toLowerCase();
+    let found = 0;
+    for (const point of mustDetectPoints) {
+        if (lower.includes(point.toLowerCase())) {
+            found++;
+        }
+    }
+    return found / mustDetectPoints.length;
+}
+/**
+ * Compute false positive rate: fraction of mustNotDetectPoints found in the output.
+ */
+function computeFalsePositiveRate(mustNotDetectPoints, searchText) {
+    if (!mustNotDetectPoints || mustNotDetectPoints.length === 0)
+        return 0.0;
+    const lower = searchText.toLowerCase();
+    let found = 0;
+    for (const point of mustNotDetectPoints) {
+        if (lower.includes(point.toLowerCase())) {
+            found++;
+        }
+    }
+    return found / mustNotDetectPoints.length;
+}
+/**
+ * Build the combined text to search for detection points.
+ * Uses all opinions from the last round's keyPoints + reasoning.
+ */
+function buildSearchText(deliberation) {
+    const lastRound = deliberation.rounds[deliberation.rounds.length - 1];
+    if (!lastRound)
+        return '';
+    const parts = [];
+    for (const opinion of lastRound.opinions) {
+        parts.push(opinion.reasoning);
+        parts.push(...opinion.keyPoints);
+        if (opinion.suggestions) {
+            parts.push(...opinion.suggestions);
+        }
+    }
+    return parts.join(' ');
+}
+/**
+ * Compute the average confidence across all opinions in the last round.
+ */
+function computeAvgConfidence(deliberation) {
+    const lastRound = deliberation.rounds[deliberation.rounds.length - 1];
+    if (!lastRound || lastRound.opinions.length === 0)
+        return 0;
+    const sum = lastRound.opinions.reduce((acc, op) => acc + op.confidence, 0);
+    return sum / lastRound.opinions.length;
+}
+/**
+ * Score a single golden task against its deliberation result.
+ *
+ * compositeScore = voteAccuracy * 0.4 + detectionCoverage * 0.5 + (1 - falsePositiveRate) * 0.1
+ */
+export function scoreTask(goldenTask, deliberation) {
+    const { expectation } = goldenTask;
+    const consensusVote = decisionToVote(deliberation.consensus.decision);
+    const voteAccuracy = consensusVote === expectation.expectedVote ? 1.0 : 0.0;
+    const searchText = buildSearchText(deliberation);
+    const detectionCoverage = computeDetectionCoverage(expectation.mustDetectPoints, searchText);
+    const falsePositiveRate = computeFalsePositiveRate(expectation.mustNotDetectPoints, searchText);
+    const compositeScore = voteAccuracy * 0.4 +
+        detectionCoverage * 0.5 +
+        (1 - falsePositiveRate) * 0.1;
+    return {
+        taskId: goldenTask.id,
+        category: goldenTask.category,
+        weight: expectation.weight,
+        voteAccuracy,
+        detectionCoverage,
+        falsePositiveRate,
+        compositeScore,
+        durationMs: deliberation.totalDurationMs,
+        avgConfidence: computeAvgConfidence(deliberation),
+    };
+}
+/**
+ * Aggregate scores across all tasks.
+ */
+export function aggregateScores(scores) {
+    if (scores.length === 0) {
+        return {
+            totalScore: 0,
+            weightedScore: 0,
+            voteAccuracy: 0,
+            detectionCoverage: 0,
+            avgDurationMs: 0,
+            totalDurationMs: 0,
+        };
+    }
+    const totalWeight = scores.reduce((acc, s) => acc + s.weight, 0);
+    const totalScore = scores.reduce((acc, s) => acc + s.compositeScore, 0) / scores.length;
+    const weightedScore = totalWeight > 0
+        ? scores.reduce((acc, s) => acc + s.compositeScore * s.weight, 0) / totalWeight
+        : 0;
+    const voteAccuracy = scores.reduce((acc, s) => acc + s.voteAccuracy, 0) / scores.length;
+    const detectionCoverage = scores.reduce((acc, s) => acc + s.detectionCoverage, 0) / scores.length;
+    const totalDurationMs = scores.reduce((acc, s) => acc + s.durationMs, 0);
+    const avgDurationMs = totalDurationMs / scores.length;
+    return {
+        totalScore,
+        weightedScore,
+        voteAccuracy,
+        detectionCoverage,
+        avgDurationMs,
+        totalDurationMs,
+    };
+}

package/dist/src/benchmark/types.d.ts

@@ -0,0 +1,54 @@
+import type { MagiTask, Vote } from '../types/core.js';
+/** Expected outcome for a golden task */
+export interface GoldenExpectation {
+    expectedVote: Vote;
+    mustDetectPoints: string[];
+    mustNotDetectPoints?: string[];
+    weight: number;
+    rationale: string;
+}
+/** A golden task with known-correct expected outcome */
+export interface GoldenTask {
+    id: string;
+    task: MagiTask;
+    expectation: GoldenExpectation;
+    category: 'code-review' | 'architecture' | 'bug-analysis' | 'security';
+}
+/** Score for a single task evaluation */
+export interface TaskScore {
+    taskId: string;
+    category: string;
+    weight: number;
+    voteAccuracy: number;
+    detectionCoverage: number;
+    falsePositiveRate: number;
+    compositeScore: number;
+    durationMs: number;
+    avgConfidence: number;
+}
+/** Aggregate scores across all tasks */
+export interface AggregateScore {
+    totalScore: number;
+    weightedScore: number;
+    voteAccuracy: number;
+    detectionCoverage: number;
+    avgDurationMs: number;
+    totalDurationMs: number;
+}
+/** Complete benchmark result for one mode */
+export interface BenchmarkResult {
+    runId: string;
+    timestamp: string;
+    gitHash: string;
+    mode: 'single-claude' | 'magi-3-body';
+    tasks: TaskScore[];
+    aggregate: AggregateScore;
+}
+/** Options for the benchmark runner */
+export interface BenchmarkOptions {
+    dryRun?: boolean;
+    singleOnly?: boolean;
+    magiOnly?: boolean;
+    category?: string;
+    outputDir?: string;
+}

package/dist/src/benchmark/types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/cache/deliberation-cache.d.ts

@@ -0,0 +1,49 @@
+import type { MagiTask, MagiDeliberation } from '../types/core.js';
+/**
+ * In-memory cache for deliberation results.
+ * Keyed by a SHA-256 hash of task attributes (type + title + description + artifacts).
+ */
+export declare class DeliberationCache {
+    private readonly ttlMs;
+    private readonly maxEntries;
+    private store;
+    private inflight;
+    constructor(ttlMs?: number, // 1 hour default
+    maxEntries?: number);
+    /**
+     * Compute a deterministic cache key from a MagiTask.
+     */
+    computeKey(task: MagiTask): string;
+    /**
+     * Get a cached deliberation result.
+     * Returns null on miss or TTL expiry.
+     * Restores Date objects and sets fromCache=true.
+     */
+    get(task: MagiTask): MagiDeliberation | null;
+    /**
+     * Store a deliberation result in the cache.
+     */
+    set(task: MagiTask, deliberation: MagiDeliberation): void;
+    /**
+     * Get from cache or compute. Prevents cache stampede by coalescing
+     * concurrent requests for the same cache key.
+     */
+    getOrCompute(task: MagiTask, compute: () => Promise<MagiDeliberation>): Promise<MagiDeliberation>;
+    /**
+     * Remove all entries from the cache.
+     */
+    clear(): void;
+    /**
+     * Remove expired entries.
+     */
+    prune(): number;
+    /**
+     * Cache statistics.
+     */
+    stats(): {
+        size: number;
+        ttlMs: number;
+        maxEntries: number;
+        inflightCount: number;
+    };
+}