magi-ai 0.1.0

package/dist/src/metrics/token-tracker.d.ts

@@ -0,0 +1,38 @@
+/** Token record for a single unit execution */
+export interface UnitTokenRecord {
+    deliberationId: string;
+    unit: string;
+    phase: string;
+    estimatedInputTokens: number;
+    estimatedOutputTokens: number;
+    timestamp: Date;
+}
+/** Summary of token usage for a deliberation */
+export interface DeliberationTokenSummary {
+    deliberationId: string;
+    totalEstimatedTokens: number;
+    byUnit: Record<string, {
+        estimatedTokens: number;
+    }>;
+    byPhase: Record<string, {
+        estimatedTokens: number;
+    }>;
+    recordCount: number;
+}
+/**
+ * Estimate token count from text.
+ * Uses a heuristic that accounts for Japanese/CJK text having fewer chars per token.
+ */
+export declare function estimateTokens(text: string): number;
+export declare class TokenTracker {
+    private records;
+    private readonly persistDir;
+    constructor(workspaceDir?: string);
+    addRecord(record: UnitTokenRecord): void;
+    summarize(deliberationId?: string): DeliberationTokenSummary;
+    persist(deliberationId: string): Promise<void>;
+    load(deliberationId: string): Promise<DeliberationTokenSummary | null>;
+    loadAll(): Promise<DeliberationTokenSummary[]>;
+    getRecords(): readonly UnitTokenRecord[];
+    clear(): void;
+}

package/dist/src/metrics/token-tracker.js

@@ -0,0 +1,112 @@
+import { readFile, readdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { UUID_V4_RE } from '../types/core.js';
+import { logger } from '../utils/logger.js';
+import { safeMkdir, safeWriteFile } from '../utils/safe-fs.js';
+import { safeJsonParse } from '../utils/safe-json-parse.js';
+/**
+ * Estimate token count from text.
+ * Uses a heuristic that accounts for Japanese/CJK text having fewer chars per token.
+ */
+export function estimateTokens(text) {
+    if (!text)
+        return 0;
+    // Count CJK characters
+    const cjkChars = (text.match(/[\u3000-\u9FFF\uF900-\uFAFF]/g) || []).length;
+    const nonCjkChars = text.length - cjkChars;
+    // CJK: ~1.5 chars per token, non-CJK: ~4 chars per token
+    return Math.ceil(cjkChars / 1.5 + nonCjkChars / 4);
+}
+export class TokenTracker {
+    records = [];
+    persistDir;
+    constructor(workspaceDir = '.magi') {
+        this.persistDir = join(workspaceDir, 'tokens');
+    }
+    addRecord(record) {
+        this.records.push(record);
+    }
+    summarize(deliberationId) {
+        const filtered = deliberationId
+            ? this.records.filter(r => r.deliberationId === deliberationId)
+            : this.records;
+        const byUnit = {};
+        const byPhase = {};
+        let totalEstimatedTokens = 0;
+        for (const r of filtered) {
+            const tokens = r.estimatedInputTokens + r.estimatedOutputTokens;
+            totalEstimatedTokens += tokens;
+            if (!byUnit[r.unit])
+                byUnit[r.unit] = { estimatedTokens: 0 };
+            byUnit[r.unit].estimatedTokens += tokens;
+            if (!byPhase[r.phase])
+                byPhase[r.phase] = { estimatedTokens: 0 };
+            byPhase[r.phase].estimatedTokens += tokens;
+        }
+        return {
+            deliberationId: deliberationId ?? 'all',
+            totalEstimatedTokens,
+            byUnit,
+            byPhase,
+            recordCount: filtered.length,
+        };
+    }
+    async persist(deliberationId) {
+        if (!UUID_V4_RE.test(deliberationId)) {
+            throw new Error('Invalid deliberation ID: must be UUID v4 format');
+        }
+        try {
+            await safeMkdir(this.persistDir);
+            const filePath = join(this.persistDir, `${deliberationId}.json`);
+            const summary = this.summarize(deliberationId);
+            await safeWriteFile(filePath, JSON.stringify(summary, null, 2));
+            logger.debug('Token data persisted', { deliberationId, file: filePath });
+        }
+        catch (err) {
+            logger.warn('Failed to persist token data', { deliberationId, error: String(err) });
+        }
+    }
+    async load(deliberationId) {
+        if (!UUID_V4_RE.test(deliberationId)) {
+            throw new Error('Invalid deliberation ID: must be UUID v4 format');
+        }
+        try {
+            const filePath = join(this.persistDir, `${deliberationId}.json`);
+            const content = await readFile(filePath, 'utf-8');
+            return safeJsonParse(content);
+        }
+        catch (err) {
+            logger.debug('TokenTracker: token data file not found', { deliberationId, error: String(err) });
+            return null;
+        }
+    }
+    async loadAll() {
+        try {
+            await safeMkdir(this.persistDir);
+            const files = await readdir(this.persistDir);
+            const summaries = [];
+            for (const file of files) {
+                if (!file.endsWith('.json'))
+                    continue;
+                try {
+                    const content = await readFile(join(this.persistDir, file), 'utf-8');
+                    summaries.push(safeJsonParse(content));
+                }
+                catch (err) {
+                    logger.debug('TokenTracker: corrupt token file skipped', { file, error: String(err) });
+                }
+            }
+            return summaries;
+        }
+        catch (err) {
+            logger.debug('TokenTracker: failed to load token summaries', { error: String(err) });
+            return [];
+        }
+    }
+    getRecords() {
+        return this.records;
+    }
+    clear() {
+        this.records = [];
+    }
+}

package/dist/src/parsers/json-extractor.d.ts

@@ -0,0 +1,9 @@
+export type ExtractionStrategy = 'strict-json' | 'envelope-unwrap' | 'markdown-block' | 'object-scan' | 'none';
+export interface ExtractionResult {
+    data: Record<string, unknown> | null;
+    raw: string;
+    effectiveRaw: string;
+    success: boolean;
+    strategy: ExtractionStrategy;
+}
+export declare function extractJson(raw: string): ExtractionResult;

package/dist/src/parsers/json-extractor.js

@@ -0,0 +1,239 @@
+/**
+ * MAGI JSON Extractor - Unified extraction module
+ *
+ * Extracts JSON objects from raw CLI output using 4 strategies:
+ * 1. strict-json    — Full text is a JSON object (with envelope guard)
+ * 2. envelope-unwrap — { type:"result", result:"..." } unwrapping
+ * 3. markdown-block  — ```json ... ``` code block extraction
+ * 4. object-scan     — Balanced brace scanning for embedded JSON
+ *
+ * Responsibility: string → Record<string, unknown> | null
+ * Downstream: opinion-schema validates the extracted object.
+ */
+import { isLikelyOpinion } from './opinion-schema.js';
+import { logger } from '../utils/logger.js';
+const ENVELOPE_KEYS = ['result', 'response', 'text', 'content', 'output'];
+function looksLikeEnvelope(obj) {
+    // Claude CLI pattern: { type: "result", result: "..." }
+    if (obj['type'] === 'result' && 'result' in obj)
+        return true;
+    // Direct opinion — not an envelope
+    if (isLikelyOpinion(obj))
+        return false;
+    // Generic envelope: known wrapper key with string-containing-JSON or nested object
+    for (const key of ENVELOPE_KEYS) {
+        const val = obj[key];
+        if (typeof val === 'string' && val.trimStart().startsWith('{'))
+            return true;
+        if (isPlainObject(val))
+            return true;
+    }
+    return false;
+}
+function isPlainObject(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+// Strategy 1: strict-json (envelope guard)
+function tryStrictJson(raw) {
+    const trimmed = raw.trim();
+    if (!trimmed.startsWith('{'))
+        return null;
+    try {
+        const parsed = JSON.parse(trimmed);
+        if (!isPlainObject(parsed))
+            return null;
+        if (looksLikeEnvelope(parsed))
+            return null;
+        return { data: parsed, effectiveRaw: raw };
+    }
+    catch (err) {
+        logger.debug('JSON extraction: strict-json parse failed', { error: String(err) });
+        return null;
+    }
+}
+// Strategy 2: envelope-unwrap
+function tryEnvelopeUnwrap(raw) {
+    const trimmed = raw.trim();
+    if (!trimmed.startsWith('{'))
+        return null;
+    let envelope;
+    try {
+        envelope = JSON.parse(trimmed);
+    }
+    catch (err) {
+        logger.debug('JSON extraction: envelope-unwrap outer parse failed', { error: String(err) });
+        return null;
+    }
+    if (!isPlainObject(envelope))
+        return null;
+    if (!looksLikeEnvelope(envelope))
+        return null;
+    for (const key of ENVELOPE_KEYS) {
+        const value = envelope[key];
+        if (typeof value === 'string') {
+            try {
+                const inner = JSON.parse(value);
+                if (isPlainObject(inner)) {
+                    return { data: inner, effectiveRaw: value };
+                }
+            }
+            catch (err) {
+                logger.debug('JSON extraction: envelope inner value not JSON', { error: String(err) });
+            }
+        }
+        if (isPlainObject(value)) {
+            return { data: value, effectiveRaw: JSON.stringify(value) };
+        }
+    }
+    return null;
+}
+/**
+ * Strip envelope wrapper and return the inner text content.
+ * Used when all extraction strategies fail but the raw text is an envelope.
+ * This prevents downstream parsers from seeing the envelope JSON as reasoning text.
+ */
+function stripEnvelopeText(raw) {
+    const trimmed = raw.trim();
+    if (!trimmed.startsWith('{'))
+        return null;
+    try {
+        const obj = JSON.parse(trimmed);
+        if (!isPlainObject(obj) || !looksLikeEnvelope(obj))
+            return null;
+        for (const key of ENVELOPE_KEYS) {
+            const value = obj[key];
+            if (typeof value === 'string' && value.trim().length > 0) {
+                return value;
+            }
+        }
+    }
+    catch {
+        // not JSON
+    }
+    return null;
+}
+// Strategy 3: markdown-block
+function tryMarkdownBlock(raw) {
+    const regex = /```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/g;
+    let match;
+    while ((match = regex.exec(raw)) !== null) {
+        const content = match[1].trim();
+        if (!content.startsWith('{'))
+            continue;
+        try {
+            const parsed = JSON.parse(content);
+            if (isPlainObject(parsed)) {
+                return { data: parsed, effectiveRaw: raw };
+            }
+        }
+        catch (err) {
+            logger.debug('JSON extraction: markdown block parse failed', { error: String(err) });
+        }
+    }
+    return null;
+}
+// Strategy 4: object-scan (balanced brace scanning)
+function findMatchingBrace(str, start) {
+    let depth = 0;
+    let inString = false;
+    let escape = false;
+    for (let i = start; i < str.length; i++) {
+        const ch = str[i];
+        if (escape) {
+            escape = false;
+            continue;
+        }
+        if (ch === '\\' && inString) {
+            escape = true;
+            continue;
+        }
+        if (ch === '"') {
+            inString = !inString;
+            continue;
+        }
+        if (inString)
+            continue;
+        if (ch === '{')
+            depth++;
+        if (ch === '}') {
+            depth--;
+            if (depth === 0)
+                return i;
+        }
+    }
+    return -1;
+}
+function extractBalancedObjects(raw) {
+    const results = [];
+    let i = 0;
+    while (i < raw.length) {
+        if (raw[i] === '{') {
+            const end = findMatchingBrace(raw, i);
+            if (end !== -1) {
+                try {
+                    const parsed = JSON.parse(raw.slice(i, end + 1));
+                    if (isPlainObject(parsed)) {
+                        results.push(parsed);
+                    }
+                }
+                catch (err) {
+                    logger.debug('JSON extraction: object-scan balanced braces but invalid JSON', { error: String(err) });
+                }
+                i = end + 1;
+                continue;
+            }
+        }
+        i++;
+    }
+    return results;
+}
+function tryObjectScan(raw) {
+    const candidates = extractBalancedObjects(raw);
+    if (candidates.length === 0)
+        return null;
+    // Filter out envelope objects — they should have been handled by envelope-unwrap.
+    // If they weren't (e.g. inner content wasn't valid JSON), returning the envelope
+    // itself would leak raw JSON into downstream reasoning fields.
+    const filtered = candidates.filter(c => !looksLikeEnvelope(c));
+    if (filtered.length === 0)
+        return null;
+    // Rank: isLikelyOpinion candidates first
+    const ranked = filtered.sort((a, b) => {
+        const aLikely = isLikelyOpinion(a) ? 0 : 1;
+        const bLikely = isLikelyOpinion(b) ? 0 : 1;
+        return aLikely - bLikely;
+    });
+    return { data: ranked[0], effectiveRaw: raw };
+}
+// Main entry point
+const strategies = [
+    { name: 'strict-json', fn: tryStrictJson },
+    { name: 'envelope-unwrap', fn: tryEnvelopeUnwrap },
+    { name: 'markdown-block', fn: tryMarkdownBlock },
+    { name: 'object-scan', fn: tryObjectScan },
+];
+export function extractJson(raw) {
+    for (const { name, fn } of strategies) {
+        const result = fn(raw);
+        if (result) {
+            return {
+                data: result.data,
+                raw,
+                effectiveRaw: result.effectiveRaw,
+                success: true,
+                strategy: name,
+            };
+        }
+    }
+    // All strategies failed. If the raw text is an envelope (e.g. Claude's
+    // {"type":"result","result":"plain text"}), strip it so downstream
+    // parsers see the inner text instead of the envelope JSON.
+    const stripped = stripEnvelopeText(raw);
+    return {
+        data: null,
+        raw,
+        effectiveRaw: stripped ?? raw,
+        success: false,
+        strategy: 'none',
+    };
+}

package/dist/src/parsers/opinion-schema.d.ts

@@ -0,0 +1,81 @@
+/**
+ * MAGI Opinion Zod Schema - Single Source of Truth
+ *
+ * All validation, normalization, and JSON Schema generation for opinion payloads.
+ * Replaces duplicated isValidOpinion()/isValidOpinionJson()/parseUnstructured()
+ * across adapters and pipelines.
+ *
+ * VoteSchema / ConfidenceSchema are canonical in core.ts and re-exported here
+ * for backward compatibility.
+ */
+import { z } from 'zod/v4';
+import type { MagiOpinion, MagiUnit } from '../types/core.js';
+import { VoteSchema, ConfidenceSchema } from '../types/core.js';
+export { VoteSchema, ConfidenceSchema };
+/** Full opinion payload schema for runtime validation */
+export declare const OpinionPayloadSchema: z.ZodObject<{
+    vote: z.ZodPipe<z.ZodPipe<z.ZodString, z.ZodTransform<string, string>>, z.ZodEnum<{
+        APPROVE: "APPROVE";
+        REJECT: "REJECT";
+        ABSTAIN: "ABSTAIN";
+    }>>;
+    confidence: z.ZodPipe<z.ZodNumber, z.ZodTransform<number, number>>;
+    reasoning: z.ZodString;
+    keyPoints: z.ZodArray<z.ZodString>;
+    suggestions: z.ZodOptional<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+/** Inferred type from the validated payload */
+export type OpinionPayload = z.infer<typeof OpinionPayloadSchema>;
+/** JSON Schema for CLI structured output (sent to Claude --json-schema) */
+export declare const OPINION_JSON_SCHEMA: Record<string, unknown>;
+/**
+ * Full validation + normalization for pipeline use.
+ * Returns normalized payload or null on failure.
+ */
+export declare function validateOpinion(obj: Record<string, unknown>): OpinionPayload | null;
+/**
+ * Lightweight check for adapter use.
+ * Only verifies vote + reasoning exist (intentionally loose).
+ * Adapters use this to identify which JSON blob is likely an opinion.
+ */
+export declare function isLikelyOpinion(obj: Record<string, unknown>): boolean;
+/**
+ * SafeOpinionSchema — derived from OpinionPayloadSchema with security constraints.
+ *
+ * Used at the orchestrator's trust boundary (where CLI output enters the system).
+ * The base OpinionPayloadSchema remains unchanged to preserve adapter semantics
+ * and the 61 existing tests.
+ *
+ * Additional constraints:
+ * - reasoning: max 10000 chars, control characters stripped
+ * - keyPoints: max 20 items, each max 1000 chars
+ * - suggestions: max 20 items, each max 1000 chars
+ * - confidence: must be finite (rejects NaN, Infinity)
+ */
+export declare const SafeOpinionSchema: z.ZodObject<{
+    vote: z.ZodPipe<z.ZodPipe<z.ZodString, z.ZodTransform<string, string>>, z.ZodEnum<{
+        APPROVE: "APPROVE";
+        REJECT: "REJECT";
+        ABSTAIN: "ABSTAIN";
+    }>>;
+    confidence: z.ZodPipe<z.ZodPipe<z.ZodNumber, z.ZodTransform<number, number>>, z.ZodNumber>;
+    reasoning: z.ZodPipe<z.ZodString, z.ZodTransform<string, string>>;
+    keyPoints: z.ZodArray<z.ZodString>;
+    suggestions: z.ZodOptional<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+export type SafeOpinionPayload = z.infer<typeof SafeOpinionSchema>;
+/**
+ * Validate opinion with safe constraints.
+ * Returns validated payload or null on failure.
+ */
+export declare function validateSafeOpinion(obj: Record<string, unknown>): SafeOpinionPayload | null;
+/**
+ * Extract opinion from unstructured text via keyword matching.
+ * Used when structured parsing fails entirely.
+ */
+export declare function parseUnstructuredOpinion(raw: string, unit: MagiUnit): Omit<MagiOpinion, 'rawOutput' | 'meta'>;
+/**
+ * Pipeline-common parse logic. Tries structured validation first,
+ * falls back to unstructured keyword extraction.
+ */
+export declare function parseOpinionFromResponse(raw: string, structured: Record<string, unknown> | null, unit: MagiUnit): Omit<MagiOpinion, 'rawOutput' | 'meta'>;

package/dist/src/parsers/opinion-schema.js

@@ -0,0 +1,147 @@
+/**
+ * MAGI Opinion Zod Schema - Single Source of Truth
+ *
+ * All validation, normalization, and JSON Schema generation for opinion payloads.
+ * Replaces duplicated isValidOpinion()/isValidOpinionJson()/parseUnstructured()
+ * across adapters and pipelines.
+ *
+ * VoteSchema / ConfidenceSchema are canonical in core.ts and re-exported here
+ * for backward compatibility.
+ */
+import { z, toJSONSchema } from 'zod/v4';
+import { VoteSchema, ConfidenceSchema } from '../types/core.js';
+import { scoreText } from './unstructured-parser.js';
+// Re-export so existing consumers of VoteSchema/ConfidenceSchema from this module still work
+export { VoteSchema, ConfidenceSchema };
+// ============================================================
+// 1. Zod Schemas (runtime validation with transforms)
+// ============================================================
+/** Full opinion payload schema for runtime validation */
+export const OpinionPayloadSchema = z.object({
+    vote: VoteSchema,
+    confidence: ConfidenceSchema,
+    reasoning: z.string().min(1),
+    keyPoints: z.array(z.string()).min(1),
+    suggestions: z.array(z.string()).optional(),
+});
+// ============================================================
+// 2. JSON Schema generation (transform-free for clean output)
+// ============================================================
+const OpinionRawSchema = z.object({
+    vote: z.enum(['APPROVE', 'REJECT', 'ABSTAIN'])
+        .describe('Your vote on this matter'),
+    confidence: z.number().min(0).max(1)
+        .describe('Confidence in your vote (0.0 to 1.0)'),
+    reasoning: z.string().min(1)
+        .describe('Detailed reasoning for your vote'),
+    keyPoints: z.array(z.string())
+        .describe('Key points supporting your position'),
+    suggestions: z.array(z.string()).optional()
+        .describe('Optional suggestions for improvement'),
+});
+/** JSON Schema for CLI structured output (sent to Claude --json-schema) */
+export const OPINION_JSON_SCHEMA = toJSONSchema(OpinionRawSchema);
+// ============================================================
+// 3. Validation functions
+// ============================================================
+/**
+ * Full validation + normalization for pipeline use.
+ * Returns normalized payload or null on failure.
+ */
+export function validateOpinion(obj) {
+    const result = OpinionPayloadSchema.safeParse(obj);
+    return result.success ? result.data : null;
+}
+/**
+ * Lightweight check for adapter use.
+ * Only verifies vote + reasoning exist (intentionally loose).
+ * Adapters use this to identify which JSON blob is likely an opinion.
+ */
+export function isLikelyOpinion(obj) {
+    return (typeof obj['vote'] === 'string' &&
+        ['APPROVE', 'REJECT', 'ABSTAIN'].includes(obj['vote'].trim().toUpperCase()) &&
+        typeof obj['reasoning'] === 'string');
+}
+// ============================================================
+// 3b. Safe opinion schema for orchestrator trust boundary
+// ============================================================
+/** Control character regex (matches \x00-\x08, \x0B, \x0C, \x0E-\x1F, \x7F) */
+const CONTROL_CHARS_RE = /[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g;
+/**
+ * SafeOpinionSchema — derived from OpinionPayloadSchema with security constraints.
+ *
+ * Used at the orchestrator's trust boundary (where CLI output enters the system).
+ * The base OpinionPayloadSchema remains unchanged to preserve adapter semantics
+ * and the 61 existing tests.
+ *
+ * Additional constraints:
+ * - reasoning: max 10000 chars, control characters stripped
+ * - keyPoints: max 20 items, each max 1000 chars
+ * - suggestions: max 20 items, each max 1000 chars
+ * - confidence: must be finite (rejects NaN, Infinity)
+ */
+export const SafeOpinionSchema = z.object({
+    vote: VoteSchema,
+    confidence: ConfidenceSchema.pipe(z.number().refine((n) => Number.isFinite(n), 'Confidence must be finite')),
+    reasoning: z
+        .string()
+        .min(1)
+        .max(10_000)
+        .transform((s) => s.replace(CONTROL_CHARS_RE, '')),
+    keyPoints: z
+        .array(z.string().max(1_000))
+        .min(1)
+        .max(20),
+    suggestions: z
+        .array(z.string().max(1_000))
+        .max(20)
+        .optional(),
+});
+/**
+ * Validate opinion with safe constraints.
+ * Returns validated payload or null on failure.
+ */
+export function validateSafeOpinion(obj) {
+    const result = SafeOpinionSchema.safeParse(obj);
+    return result.success ? result.data : null;
+}
+// ============================================================
+// 4. Unstructured text fallback parser
+// ============================================================
+/**
+ * Extract opinion from unstructured text via keyword matching.
+ * Used when structured parsing fails entirely.
+ */
+export function parseUnstructuredOpinion(raw, unit) {
+    const result = scoreText(raw);
+    return {
+        unit,
+        vote: result.vote,
+        confidence: result.confidence,
+        reasoning: raw.slice(0, 2000),
+        keyPoints: ['(Parsed from unstructured output)'],
+    };
+}
+// ============================================================
+// 5. Unified parse: structured → validate → fallback
+// ============================================================
+/**
+ * Pipeline-common parse logic. Tries structured validation first,
+ * falls back to unstructured keyword extraction.
+ */
+export function parseOpinionFromResponse(raw, structured, unit) {
+    if (structured) {
+        const validated = validateOpinion(structured);
+        if (validated) {
+            return {
+                unit,
+                vote: validated.vote,
+                confidence: validated.confidence,
+                reasoning: validated.reasoning,
+                keyPoints: validated.keyPoints,
+                suggestions: validated.suggestions,
+            };
+        }
+    }
+    return parseUnstructuredOpinion(raw, unit);
+}

package/dist/src/parsers/unstructured-parser.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Weighted scoring engine for unstructured text → vote extraction.
+ *
+ * Fixes the false-positive bug where "I don't approve" was classified as APPROVE.
+ * Uses 22 patterns with negation-aware matching, weighted scoring,
+ * and dynamic confidence extraction.
+ */
+import type { Vote } from '../types/core.js';
+export interface ScoreResult {
+    vote: Vote;
+    confidence: number;
+    _debug: {
+        approveScore: number;
+        rejectScore: number;
+        abstainScore: number;
+        extractedConfidence: number | null;
+        matchedPatterns: string[];
+    };
+}
+export declare function scoreText(raw: string): ScoreResult;