magi-ai 0.1.0

package/dist/src/audit/writer.js

@@ -0,0 +1,100 @@
+import { readFile, rename, readdir, unlink, stat } from 'node:fs/promises';
+import { join } from 'node:path';
+import { computeEntryHash } from './hash-chain.js';
+import { logger } from '../utils/logger.js';
+import { safeMkdir, safeAppendFile } from '../utils/safe-fs.js';
+import { safeJsonParse } from '../utils/safe-json-parse.js';
+const DEFAULT_MAX_FILE_SIZE = 10 * 1024 * 1024; // 10MB
+const DEFAULT_MAX_FILES = 5;
+const LOG_FILE_NAME = 'magi-audit.jsonl';
+export class AuditLogWriter {
+    config;
+    lastHash = '';
+    currentFile;
+    currentSize = 0;
+    writeQueue = Promise.resolve();
+    constructor(config) {
+        this.config = config;
+        this.currentFile = join(config.logDir, LOG_FILE_NAME);
+    }
+    /** Initialize: create log directory with 0o700, set current file, recover lastHash from existing log */
+    async initialize() {
+        await safeMkdir(this.config.logDir);
+        try {
+            const content = await readFile(this.currentFile, 'utf-8');
+            const lines = content.trimEnd().split('\n');
+            const lastLine = lines[lines.length - 1];
+            if (lastLine) {
+                const lastEntry = safeJsonParse(lastLine);
+                this.lastHash = lastEntry.hash;
+            }
+            const stats = await stat(this.currentFile);
+            this.currentSize = stats.size;
+        }
+        catch (err) {
+            logger.debug('AuditLogWriter: log file not found, starting fresh', { error: String(err) });
+            this.lastHash = '';
+            this.currentSize = 0;
+        }
+    }
+    /** Write a single entry, auto-rotate if needed. Serializes concurrent calls. */
+    async write(event, deliberationId, data) {
+        return new Promise((resolve, reject) => {
+            this.writeQueue = this.writeQueue.then(async () => {
+                try {
+                    const entry = await this.writeEntry(event, deliberationId, data);
+                    resolve(entry);
+                }
+                catch (err) {
+                    reject(err);
+                }
+            });
+        });
+    }
+    /** Get the current log file path */
+    get logFile() {
+        return this.currentFile;
+    }
+    async writeEntry(event, deliberationId, data) {
+        const maxSize = this.config.maxFileSize ?? DEFAULT_MAX_FILE_SIZE;
+        if (this.currentSize > 0 && this.currentSize >= maxSize) {
+            await this.rotate();
+        }
+        const partial = {
+            timestamp: new Date().toISOString(),
+            event,
+            deliberationId,
+            data,
+            prevHash: this.lastHash,
+        };
+        const hash = computeEntryHash(partial);
+        const entry = { ...partial, hash };
+        const line = JSON.stringify(entry) + '\n';
+        await safeAppendFile(this.currentFile, line);
+        this.lastHash = hash;
+        this.currentSize += Buffer.byteLength(line, 'utf-8');
+        return entry;
+    }
+    /** Rotate: rename current to timestamped, clean old files beyond maxFiles */
+    async rotate() {
+        const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+        const rotatedName = `magi-audit-${timestamp}.jsonl`;
+        const rotatedPath = join(this.config.logDir, rotatedName);
+        await rename(this.currentFile, rotatedPath);
+        // Reset state for new file
+        this.currentSize = 0;
+        this.lastHash = '';
+        // Clean old rotated files if exceeding maxFiles
+        const maxFiles = this.config.maxFiles ?? DEFAULT_MAX_FILES;
+        const allFiles = await readdir(this.config.logDir);
+        const rotatedFiles = allFiles
+            .filter((f) => f.startsWith('magi-audit-') && f.endsWith('.jsonl'))
+            .sort();
+        if (rotatedFiles.length > maxFiles) {
+            const toDelete = rotatedFiles.slice(0, rotatedFiles.length - maxFiles);
+            for (const file of toDelete) {
+                await unlink(join(this.config.logDir, file));
+            }
+        }
+    }
+}

package/dist/src/benchmark/golden-tasks.d.ts

@@ -0,0 +1,9 @@
+import type { GoldenTask } from './types.js';
+/**
+ * 16 golden tasks for benchmarking MAGI quality.
+ *
+ * - APPROVE tasks (3): false-positive detection
+ * - REJECT tasks (13): true-positive detection
+ * - Security tasks use type: 'code-review' (no SecurityAuditPipeline yet)
+ */
+export declare const GOLDEN_TASKS: readonly GoldenTask[];

package/dist/src/benchmark/golden-tasks.js

@@ -0,0 +1,476 @@
+/**
+ * 16 golden tasks for benchmarking MAGI quality.
+ *
+ * - APPROVE tasks (3): false-positive detection
+ * - REJECT tasks (13): true-positive detection
+ * - Security tasks use type: 'code-review' (no SecurityAuditPipeline yet)
+ */
+export const GOLDEN_TASKS = [
+    // ── Code Review ──────────────────────────────────────────────
+    {
+        id: 'cr-001',
+        category: 'code-review',
+        task: {
+            type: 'code-review',
+            title: 'Off-by-one error in slice boundary',
+            description: 'Review the following code for correctness issues.',
+            artifacts: [{
+                    type: 'snippet',
+                    language: 'typescript',
+                    content: `function getLastN<T>(arr: T[], n: number): T[] {
+  // Should return last n elements
+  return arr.slice(arr.length - n - 1);
+}`,
+                }],
+        },
+        expectation: {
+            expectedVote: 'REJECT',
+            mustDetectPoints: ['off-by-one', 'slice', 'boundary', 'off by one'],
+            weight: 1.0,
+            rationale: 'slice(arr.length - n - 1) returns n+1 elements instead of n',
+        },
+    },
+    {
+        id: 'cr-002',
+        category: 'code-review',
+        task: {
+            type: 'code-review',
+            title: 'Race condition in check-then-act pattern',
+            description: 'Review the following code for concurrency issues.',
+            artifacts: [{
+                    type: 'snippet',
+                    language: 'typescript',
+                    content: `import { existsSync, writeFileSync, readFileSync } from 'fs';
+
+async function safeWrite(path: string, data: string): Promise<void> {
+  if (!existsSync(path)) {
+    writeFileSync(path, data);
+  } else {
+    const existing = readFileSync(path, 'utf-8');
+    writeFileSync(path, existing + data);
+  }
+}`,
+                }],
+        },
+        expectation: {
+            expectedVote: 'REJECT',
+            mustDetectPoints: ['race condition', 'TOCTOU', 'check-then-act', 'time-of-check', 'concurrent', 'data race'],
+            weight: 1.5,
+            rationale: 'TOCTOU race: file can be created/modified between existsSync and writeFileSync',
+        },
+    },
+    {
+        id: 'cr-003',
+        category: 'code-review',
+        task: {
+            type: 'code-review',
+            title: 'Memory leak from unremoved event listener',
+            description: 'Review the following React component for resource management issues.',
+            artifacts: [{
+                    type: 'snippet',
+                    language: 'typescript',
+                    content: `import { useEffect, useState } from 'react';
+
+function useWindowSize() {
+  const [size, setSize] = useState({ w: 0, h: 0 });
+
+  useEffect(() => {
+    const handler = () => setSize({ w: window.innerWidth, h: window.innerHeight });
+    window.addEventListener('resize', handler);
+    // Missing cleanup: return () => window.removeEventListener('resize', handler);
+  }, []);
+
+  return size;
+}`,
+                }],
+        },
+        expectation: {
+            expectedVote: 'REJECT',
+            mustDetectPoints: ['memory leak', 'event listener', 'cleanup', 'removeEventListener', 'unmount'],
+            weight: 1.0,
+            rationale: 'useEffect never removes the resize listener, causing memory leak on unmount',
+        },
+    },
+    {
+        id: 'cr-004',
+        category: 'code-review',
+        task: {
+            type: 'code-review',
+            title: 'Null reference without optional chaining',
+            description: 'Review the following code for null safety issues.',
+            artifacts: [{
+                    type: 'snippet',
+                    language: 'typescript',
+                    content: `interface User {
+  profile?: {
+    address?: {
+      city: string;
+    };
+  };
+}
+
+function getCity(user: User): string {
+  return user.profile.address.city;
+}`,
+                }],
+        },
+        expectation: {
+            expectedVote: 'REJECT',
+            mustDetectPoints: ['null', 'undefined', 'optional chaining', 'TypeError', 'optional'],
+            weight: 1.0,
+            rationale: 'Accessing .profile.address.city without optional chaining throws when profile/address is undefined',
+        },
+    },
+    {
+        id: 'cr-005',
+        category: 'code-review',
+        task: {
+            type: 'code-review',
+            title: 'Well-implemented Result type pattern',
+            description: 'Review the following utility code for correctness.',
+            artifacts: [{
+                    type: 'snippet',
+                    language: 'typescript',
+                    content: `type Result<T, E = Error> =
+  | { ok: true; value: T }
+  | { ok: false; error: E };
+
+function tryParse(json: string): Result<unknown> {
+  try {
+    return { ok: true, value: JSON.parse(json) };
+  } catch (e) {
+    return { ok: false, error: e instanceof Error ? e : new Error(String(e)) };
+  }
+}
+
+function unwrap<T>(result: Result<T>): T {
+  if (!result.ok) throw result.error;
+  return result.value;
+}`,
+                }],
+        },
+        expectation: {
+            expectedVote: 'APPROVE',
+            mustDetectPoints: ['Result', 'type-safe', 'error handling', 'pattern'],
+            mustNotDetectPoints: ['vulnerability', 'critical bug', 'security risk'],
+            weight: 1.0,
+            rationale: 'Clean Result type pattern with proper error handling — should be approved',
+        },
+    },
+    // ── Architecture ─────────────────────────────────────────────
+    {
+        id: 'arch-001',
+        category: 'architecture',
+        task: {
+            type: 'architecture-decision',
+            title: 'Monolith vs Microservices for MVP',
+            description: `We are building an internal tool for a team of 5 developers.
+Expected users: ~50. Expected traffic: ~100 requests/day.
+Should we use a monolithic architecture or microservices?
+
+Proposal: Use a single Node.js + PostgreSQL monolith with modular internal structure.`,
+        },
+        expectation: {
+            expectedVote: 'APPROVE',
+            mustDetectPoints: ['monolith', 'simple', 'appropriate', 'scale'],
+            mustNotDetectPoints: ['microservices required'],
+            weight: 1.0,
+            rationale: 'Monolith is the right choice for a small team/low traffic MVP',
+        },
+    },
+    {
+        id: 'arch-002',
+        category: 'architecture',
+        task: {
+            type: 'architecture-decision',
+            title: 'Tightly coupled design with circular dependencies',
+            description: `Review this proposed module structure:
+
+- UserService imports OrderService
+- OrderService imports PaymentService
+- PaymentService imports UserService (for user credit check)
+- All services share a single mutable state object
+
+Each service directly instantiates its dependencies with \`new\`.`,
+        },
+        expectation: {
+            expectedVote: 'REJECT',
+            mustDetectPoints: ['circular dependency', 'tight coupling', 'coupled', 'circular', 'dependency injection', 'mutable state'],
+            weight: 1.5,
+            rationale: 'Circular dependencies + shared mutable state + no DI = unmaintainable',
+        },
+    },
+    {
+        id: 'arch-003',
+        category: 'architecture',
+        task: {
+            type: 'architecture-decision',
+            title: 'Singleton Logger implementation',
+            description: `Proposal: Implement a singleton logger for the application.
+
+\`\`\`typescript
+class Logger {
+  private static instance: Logger;
+  private constructor(private level: string) {}
+  static getInstance(): Logger {
+    if (!Logger.instance) Logger.instance = new Logger('info');
+    return Logger.instance;
+  }
+  info(msg: string) { console.log(\`[INFO] \${msg}\`); }
+  error(msg: string) { console.error(\`[ERROR] \${msg}\`); }
+}
+\`\`\`
+
+This will be used as the standard logging mechanism across the application.`,
+        },
+        expectation: {
+            expectedVote: 'APPROVE',
+            mustDetectPoints: ['singleton', 'logger', 'pattern', 'acceptable'],
+            weight: 1.0,
+            rationale: 'Singleton logger is a well-established, acceptable pattern',
+        },
+    },
+    {
+        id: 'arch-004',
+        category: 'architecture',
+        task: {
+            type: 'architecture-decision',
+            title: 'Over-abstracted factory pattern (YAGNI violation)',
+            description: `Proposal for a configuration system:
+
+- AbstractConfigProvider (base class)
+- ConfigProviderFactory (creates providers)
+- ConfigProviderRegistry (registers factories)
+- ConfigProviderAdapter (adapts between formats)
+- ConfigProviderProxy (lazy loading)
+- ConfigProviderDecorator (adds caching)
+
+Currently we only have one config source: a JSON file.
+No plans for additional sources in the roadmap.`,
+        },
+        expectation: {
+            expectedVote: 'REJECT',
+            mustDetectPoints: ['YAGNI', 'over-engineer', 'abstraction', 'complex', 'unnecessary', 'overdesign'],
+            weight: 1.0,
+            rationale: '6 classes for one JSON file is extreme over-engineering',
+        },
+    },
+    {
+        id: 'arch-005',
+        category: 'architecture',
+        task: {
+            type: 'architecture-decision',
+            title: 'API design missing error handling',
+            description: `Proposed REST API endpoints:
+
+- POST /api/users — creates user, returns 200 with user object
+- GET /api/users/:id — returns user or empty body
+- PUT /api/users/:id — updates user, returns 200
+- DELETE /api/users/:id — deletes user, returns 200
+
+No error response format defined. No validation middleware.
+No rate limiting. No authentication mentioned.`,
+        },
+        expectation: {
+            expectedVote: 'REJECT',
+            mustDetectPoints: ['error handling', 'validation', 'authentication', 'status code', 'error response'],
+            weight: 1.0,
+            rationale: 'API design lacks error responses, validation, auth, and proper status codes',
+        },
+    },
+    // ── Bug Analysis ─────────────────────────────────────────────
+    {
+        id: 'bug-001',
+        category: 'bug-analysis',
+        task: {
+            type: 'bug-analysis',
+            title: 'Infinite recursion causing stack overflow',
+            description: 'Investigate the following code that crashes with "Maximum call stack size exceeded".',
+            artifacts: [{
+                    type: 'snippet',
+                    language: 'typescript',
+                    content: `interface TreeNode {
+  value: number;
+  children: TreeNode[];
+}
+
+function flatten(node: TreeNode): number[] {
+  const result = [node.value];
+  for (const child of node.children) {
+    result.push(...flatten(child));
+  }
+  return result;
+}
+
+// Bug: circular reference
+const root: TreeNode = { value: 1, children: [] };
+const child: TreeNode = { value: 2, children: [root] }; // circular!
+root.children.push(child);
+flatten(root); // Stack overflow`,
+                }],
+        },
+        expectation: {
+            expectedVote: 'REJECT',
+            mustDetectPoints: ['infinite recursion', 'stack overflow', 'circular', 'cycle', 'recursive'],
+            weight: 1.0,
+            rationale: 'Circular reference causes infinite recursion in flatten()',
+        },
+    },
+    {
+        id: 'bug-002',
+        category: 'bug-analysis',
+        task: {
+            type: 'bug-analysis',
+            title: 'Swallowed async error in Promise chain',
+            description: 'Investigate why errors from this async operation are silently lost.',
+            artifacts: [{
+                    type: 'snippet',
+                    language: 'typescript',
+                    content: `async function fetchData(url: string): Promise<unknown> {
+  try {
+    const response = await fetch(url);
+    const data = await response.json();
+    return data;
+  } catch {
+    // Error silently swallowed — no logging, no rethrow, no fallback
+    return undefined;
+  }
+}
+
+async function processAll(urls: string[]) {
+  const results = urls.map(url => fetchData(url));
+  // Missing await — fires and forgets
+  Promise.all(results);
+}`,
+                }],
+        },
+        expectation: {
+            expectedVote: 'REJECT',
+            mustDetectPoints: ['swallow', 'error handling', 'catch', 'silent', 'await', 'fire and forget'],
+            weight: 1.0,
+            rationale: 'Empty catch block swallows errors; Promise.all result not awaited',
+        },
+    },
+    {
+        id: 'bug-003',
+        category: 'bug-analysis',
+        task: {
+            type: 'bug-analysis',
+            title: 'Sort comparator treating numbers as strings',
+            description: 'Investigate why sorting produces incorrect results for numeric arrays.',
+            artifacts: [{
+                    type: 'snippet',
+                    language: 'typescript',
+                    content: `function sortNumbers(arr: number[]): number[] {
+  return [...arr].sort();
+  // Default sort converts to strings: [1, 10, 2, 20, 3]
+  // Should be: arr.sort((a, b) => a - b)
+}
+
+// Example: sortNumbers([10, 1, 20, 2, 3])
+// Expected: [1, 2, 3, 10, 20]
+// Actual:   [1, 10, 2, 20, 3]`,
+                }],
+        },
+        expectation: {
+            expectedVote: 'REJECT',
+            mustDetectPoints: ['sort', 'string comparison', 'lexicographic', 'comparator', 'numeric'],
+            weight: 1.0,
+            rationale: 'Default Array.sort() uses lexicographic comparison, not numeric',
+        },
+    },
+    // ── Security (using code-review type) ────────────────────────
+    {
+        id: 'sec-001',
+        category: 'security',
+        task: {
+            type: 'code-review',
+            title: 'SQL injection vulnerability',
+            description: 'Security review of database query code.',
+            artifacts: [{
+                    type: 'snippet',
+                    language: 'typescript',
+                    content: `import { Pool } from 'pg';
+
+const pool = new Pool();
+
+async function getUser(userId: string) {
+  const query = \`SELECT * FROM users WHERE id = '\${userId}'\`;
+  const result = await pool.query(query);
+  return result.rows[0];
+}
+
+// Called with user input from HTTP request:
+// app.get('/user/:id', (req, res) => getUser(req.params.id))`,
+                }],
+        },
+        expectation: {
+            expectedVote: 'REJECT',
+            mustDetectPoints: ['SQL injection', 'parameterized', 'prepared statement', 'sanitize', 'injection'],
+            weight: 2.0,
+            rationale: 'Direct string interpolation in SQL query allows SQL injection',
+        },
+    },
+    {
+        id: 'sec-002',
+        category: 'security',
+        task: {
+            type: 'code-review',
+            title: 'XSS via innerHTML',
+            description: 'Security review of DOM manipulation code.',
+            artifacts: [{
+                    type: 'snippet',
+                    language: 'typescript',
+                    content: `function renderComment(comment: { author: string; text: string }) {
+  const container = document.getElementById('comments');
+  if (container) {
+    container.innerHTML += \`
+      <div class="comment">
+        <strong>\${comment.author}</strong>
+        <p>\${comment.text}</p>
+      </div>
+    \`;
+  }
+}
+
+// comment.text comes directly from user input via form submission`,
+                }],
+        },
+        expectation: {
+            expectedVote: 'REJECT',
+            mustDetectPoints: ['XSS', 'innerHTML', 'cross-site scripting', 'sanitize', 'escape', 'script injection'],
+            weight: 2.0,
+            rationale: 'User input inserted via innerHTML without escaping enables XSS',
+        },
+    },
+    {
+        id: 'sec-003',
+        category: 'security',
+        task: {
+            type: 'code-review',
+            title: 'Path traversal vulnerability',
+            description: 'Security review of file serving code.',
+            artifacts: [{
+                    type: 'snippet',
+                    language: 'typescript',
+                    content: `import { readFileSync } from 'fs';
+import { join } from 'path';
+
+function serveFile(userPath: string): string {
+  const filePath = join('/app/public', userPath);
+  return readFileSync(filePath, 'utf-8');
+}
+
+// Called from HTTP handler:
+// app.get('/files/*', (req, res) => res.send(serveFile(req.params[0])))
+// Attacker sends: GET /files/../../etc/passwd`,
+                }],
+        },
+        expectation: {
+            expectedVote: 'REJECT',
+            mustDetectPoints: ['path traversal', 'directory traversal', '../', 'resolve', 'normalize', 'sanitize path'],
+            weight: 2.0,
+            rationale: 'join() does not prevent ../ traversal; attacker can read arbitrary files',
+        },
+    },
+];

package/dist/src/benchmark/reporter.d.ts

@@ -0,0 +1,5 @@
+import type { BenchmarkResult } from './types.js';
+/**
+ * Generate a Markdown benchmark report comparing single-Claude vs MAGI 3-body results.
+ */
+export declare function generateReport(singleResult: BenchmarkResult | null, magiResult: BenchmarkResult | null): string;