@wooojin/forgen 0.3.0 → 0.3.2

package/dist/engine/solution-outcomes.js

@@ -0,0 +1,319 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { OUTCOMES_DIR, STATE_DIR } from '../core/paths.js';
+import { sanitizeId } from '../hooks/shared/sanitize-id.js';
+import { withFileLockSync, FileLockError } from '../hooks/shared/file-lock.js';
+import { atomicWriteJSON } from '../hooks/shared/atomic-write.js';
+import { createLogger } from '../core/logger.js';
+const log = createLogger('solution-outcomes');
+function pendingPath(sessionId) {
+    return path.join(STATE_DIR, `outcome-pending-${sanitizeId(sessionId)}.json`);
+}
+function outcomesPath(sessionId) {
+    return path.join(OUTCOMES_DIR, `${sanitizeId(sessionId)}.jsonl`);
+}
+function readPending(sessionId) {
+    const p = pendingPath(sessionId);
+    if (!fs.existsSync(p))
+        return { pending: [], last_prompt_ts: 0 };
+    try {
+        return JSON.parse(fs.readFileSync(p, 'utf-8'));
+    }
+    catch {
+        return { pending: [], last_prompt_ts: 0 };
+    }
+}
+function writePending(sessionId, state) {
+    const p = pendingPath(sessionId);
+    fs.mkdirSync(STATE_DIR, { recursive: true });
+    atomicWriteJSON(p, state, { mode: 0o600, dirMode: 0o700 });
+}
+function appendOutcome(event) {
+    fs.mkdirSync(OUTCOMES_DIR, { recursive: true });
+    fs.appendFileSync(outcomesPath(event.session_id), JSON.stringify(event) + '\n');
+}
+/**
+ * Run a read-modify-write pending-state mutation under a file lock
+ * (2026-04-21 audit fix #9).
+ *
+ * Prior code (`readPending` → in-memory mutate → `writePending`) was not
+ * serialised, so inject + correction + error hooks racing on the same
+ * session could lose or duplicate outcome events. The entire critical
+ * section now sits inside `withFileLockSync`; the lock file lives next
+ * to the pending file itself. A lock-acquire failure falls through
+ * fail-open (outcome tracking must never block the user).
+ */
+function mutatePending(sessionId, fn) {
+    const p = pendingPath(sessionId);
+    fs.mkdirSync(STATE_DIR, { recursive: true });
+    try {
+        let result = null;
+        withFileLockSync(p, () => {
+            result = fn();
+        });
+        return result;
+    }
+    catch (e) {
+        if (e instanceof FileLockError) {
+            log.debug(`pending lock 실패 — skip (fail-open): ${e.message}`);
+            return null;
+        }
+        throw e;
+    }
+}
+/**
+ * Record that solutions were injected. Called from solution-injector right
+ * after `approveWithContext` is emitted. Fails silently — outcome tracking
+ * must never block the user's workflow.
+ */
+export function appendPending(sessionId, injections) {
+    if (!sessionId || injections.length === 0)
+        return;
+    try {
+        mutatePending(sessionId, () => {
+            const state = readPending(sessionId);
+            const ts = Date.now();
+            for (const inj of injections) {
+                state.pending.push({ ...inj, ts });
+            }
+            writePending(sessionId, state);
+        });
+    }
+    catch (e) {
+        log.debug(`appendPending failed: ${e instanceof Error ? e.message : String(e)}`);
+    }
+}
+/**
+ * Flush pending injections as `accept` events. Called when a new user
+ * prompt arrives without any intervening correction/error, signaling that
+ * the previous injections were silently accepted. "Silence = consent."
+ *
+ * If `excludeSolutions` is provided, those solutions are NOT flushed (e.g.
+ * because an earlier step already attributed them as `correct` or `error`).
+ */
+export function flushAccept(sessionId, excludeSolutions = new Set()) {
+    if (!sessionId)
+        return 0;
+    try {
+        const result = mutatePending(sessionId, () => {
+            const state = readPending(sessionId);
+            if (state.pending.length === 0)
+                return 0;
+            const now = Date.now();
+            const kept = [];
+            let flushed = 0;
+            for (const p of state.pending) {
+                // P1-L1 fix (2026-04-20): 이전에는 excluded pending을 `continue`로 건너뛰면서
+                // `kept`에도 push 안 하고 appendOutcome도 안 해서 증거 없이 사라졌다.
+                // 이미 correct/error로 attribute된 항목은 보존 (나중 prompt에서 재처리 방지).
+                if (excludeSolutions.has(p.solution)) {
+                    kept.push(p);
+                    continue;
+                }
+                appendOutcome({
+                    ts: now,
+                    session_id: sessionId,
+                    solution: p.solution,
+                    match_score: p.match_score,
+                    injected_chars: p.injected_chars,
+                    outcome: 'accept',
+                    outcome_lag_ms: now - p.ts,
+                    attribution: 'default',
+                });
+                flushed++;
+            }
+            writePending(sessionId, { pending: kept, last_prompt_ts: now });
+            return flushed;
+        });
+        return result ?? 0;
+    }
+    catch (e) {
+        log.debug(`flushAccept failed: ${e instanceof Error ? e.message : String(e)}`);
+        return 0;
+    }
+}
+/**
+ * Attribute a correction to the most recent pending injection(s). Called
+ * from the correction-record MCP tool. Removes attributed entries from
+ * pending so subsequent `flushAccept` does not double-count them.
+ *
+ * Strategy: all currently-pending solutions in this session are marked as
+ * `correct`. This is conservative (the correction may target only one of
+ * them), but without semantic attribution we err on the side of the user's
+ * feedback signal being louder than acceptance.
+ */
+export function attributeCorrection(sessionId) {
+    if (!sessionId)
+        return [];
+    try {
+        const result = mutatePending(sessionId, () => {
+            const state = readPending(sessionId);
+            if (state.pending.length === 0)
+                return [];
+            const now = Date.now();
+            const attributed = [];
+            for (const p of state.pending) {
+                appendOutcome({
+                    ts: now,
+                    session_id: sessionId,
+                    solution: p.solution,
+                    match_score: p.match_score,
+                    injected_chars: p.injected_chars,
+                    outcome: 'correct',
+                    outcome_lag_ms: now - p.ts,
+                    attribution: 'explicit',
+                });
+                attributed.push(p.solution);
+            }
+            writePending(sessionId, { pending: [], last_prompt_ts: state.last_prompt_ts });
+            return attributed;
+        });
+        return result ?? [];
+    }
+    catch (e) {
+        log.debug(`attributeCorrection failed: ${e instanceof Error ? e.message : String(e)}`);
+        return [];
+    }
+}
+/**
+ * Attribution gates for error events (2026-04-21 fix):
+ *   - MIN_ERROR_MATCH_SCORE: solutions injected on thin relevance (< 0.3)
+ *     are unlikely to have caused unrelated tool failures. Attributing
+ *     error to them distorts fitness and poisons the Phase 4 evolver.
+ *   - MAX_ATTRIBUTION_LAG_MS: errors more than 5 minutes after injection
+ *     are too temporally distant — the user has likely moved on to work
+ *     the solution has no bearing on.
+ *   - MAX_ATTRIBUTED_PER_ERROR: cap at the top 3 most-relevant pending
+ *     solutions so a broad inject batch doesn't scatter false blame.
+ *
+ * Data audit (2026-04-21, ~/.forgen/state/outcomes/ 260 sessions):
+ *   Top-3 "error" solutions = 80% of all error events, injected at score
+ *   0.15/0.21 nearly every session. Without these gates, fitness
+ *   classification drags good-but-low-score solutions into underperform.
+ */
+const MIN_ERROR_MATCH_SCORE = 0.3;
+const MAX_ATTRIBUTION_LAG_MS = 5 * 60 * 1000;
+const MAX_ATTRIBUTED_PER_ERROR = 3;
+/**
+ * Attribute a tool error to pending solutions in this session. Called from
+ * post-tool-failure hook. Unlike corrections, errors do not clear pending
+ * — an error is a weaker signal and the next user prompt can still produce
+ * a correct/accept decision.
+ *
+ * Only the top-K most-relevant, recent, above-threshold pending solutions
+ * are attributed (see gates above). Below-threshold or stale pending
+ * entries are left untouched — they will resolve via accept/unknown later.
+ *
+ * To avoid flooding the log with duplicate errors for the same pending
+ * batch, we cap at one `error` event per (session, solution) pair per
+ * pending-cycle by tracking a `error_flagged` set in the pending state.
+ */
+export function attributeError(sessionId) {
+    if (!sessionId)
+        return [];
+    try {
+        const result = mutatePending(sessionId, () => {
+            const state = readPending(sessionId);
+            if (state.pending.length === 0)
+                return [];
+            const flaggedKey = `__error_flagged`;
+            const existing = state[flaggedKey];
+            const flagged = new Set(Array.isArray(existing) ? existing : []);
+            const now = Date.now();
+            const eligible = state.pending
+                .filter((p) => !flagged.has(p.solution))
+                .filter((p) => p.match_score >= MIN_ERROR_MATCH_SCORE)
+                .filter((p) => now - p.ts <= MAX_ATTRIBUTION_LAG_MS)
+                .sort((a, b) => b.match_score - a.match_score)
+                .slice(0, MAX_ATTRIBUTED_PER_ERROR);
+            const flaggedThisCall = [];
+            for (const p of eligible) {
+                appendOutcome({
+                    ts: now,
+                    session_id: sessionId,
+                    solution: p.solution,
+                    match_score: p.match_score,
+                    injected_chars: p.injected_chars,
+                    outcome: 'error',
+                    outcome_lag_ms: now - p.ts,
+                    attribution: 'window',
+                });
+                flagged.add(p.solution);
+                flaggedThisCall.push(p.solution);
+            }
+            state[flaggedKey] = Array.from(flagged);
+            writePending(sessionId, state);
+            return flaggedThisCall;
+        });
+        return result ?? [];
+    }
+    catch (e) {
+        log.debug(`attributeError failed: ${e instanceof Error ? e.message : String(e)}`);
+        return [];
+    }
+}
+/**
+ * At session end, any still-pending entries are logged as `unknown` (we
+ * can't tell if the user was happy or just stopped). Pending file is
+ * removed.
+ */
+export function finalizeSession(sessionId) {
+    if (!sessionId)
+        return 0;
+    try {
+        const result = mutatePending(sessionId, () => {
+            const state = readPending(sessionId);
+            const now = Date.now();
+            let finalized = 0;
+            for (const p of state.pending) {
+                appendOutcome({
+                    ts: now,
+                    session_id: sessionId,
+                    solution: p.solution,
+                    match_score: p.match_score,
+                    injected_chars: p.injected_chars,
+                    outcome: 'unknown',
+                    outcome_lag_ms: now - p.ts,
+                    attribution: 'session_end',
+                });
+                finalized++;
+            }
+            const pendingFile = pendingPath(sessionId);
+            if (fs.existsSync(pendingFile))
+                fs.unlinkSync(pendingFile);
+            return finalized;
+        });
+        return result ?? 0;
+    }
+    catch (e) {
+        log.debug(`finalizeSession failed: ${e instanceof Error ? e.message : String(e)}`);
+        return 0;
+    }
+}
+/**
+ * Read all outcome events across all sessions. Used by fitness
+ * calculation. Returns events sorted by timestamp ascending.
+ */
+export function readAllOutcomes() {
+    if (!fs.existsSync(OUTCOMES_DIR))
+        return [];
+    const events = [];
+    for (const file of fs.readdirSync(OUTCOMES_DIR)) {
+        if (!file.endsWith('.jsonl'))
+            continue;
+        try {
+            const text = fs.readFileSync(path.join(OUTCOMES_DIR, file), 'utf-8');
+            for (const line of text.split('\n')) {
+                if (!line)
+                    continue;
+                try {
+                    events.push(JSON.parse(line));
+                }
+                catch { /* skip bad line */ }
+            }
+        }
+        catch { /* skip */ }
+    }
+    events.sort((a, b) => a.ts - b.ts);
+    return events;
+}

package/dist/engine/solution-quarantine.d.ts

@@ -0,0 +1,36 @@
+interface QuarantineEntry {
+    path: string;
+    at: string;
+    errors: string[];
+}
+/**
+ * Produce actionable frontmatter diagnostics directly from file content.
+ *
+ * This duplicates the YAML parse that `parseFrontmatterOnly` already does,
+ * but it runs only on the rare failure path (solution dropped from index),
+ * so the overhead is acceptable in exchange for a human-readable error list.
+ */
+export declare function diagnoseFromRawContent(content: string): string[];
+/**
+ * Append one quarantine entry for `filePath`. Deduped by path within the
+ * current file: if the latest entry for this path already matches the
+ * current errors, skip the append.
+ *
+ * Storage: one JSONL line per quarantine event. Readers use only the
+ * latest line per path.
+ */
+export declare function recordQuarantine(filePath: string, errors: string[]): void;
+/**
+ * Read the latest quarantine state: one entry per path, keyed to the most
+ * recent append. Entries whose file no longer exists are dropped.
+ */
+export declare function listQuarantined(): QuarantineEntry[];
+/**
+ * Clear quarantine entries for files that now parse correctly or no longer
+ * exist. Intended to be called after `forgen learn fix-up` or a manual edit.
+ */
+export declare function pruneQuarantine(): {
+    removed: number;
+    kept: number;
+};
+export {};

package/dist/engine/solution-quarantine.js

@@ -0,0 +1,172 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import yaml from 'js-yaml';
+import { SOLUTION_QUARANTINE_PATH, STATE_DIR } from '../core/paths.js';
+import { diagnoseFrontmatter } from './solution-format.js';
+import { createLogger } from '../core/logger.js';
+const log = createLogger('solution-quarantine');
+/**
+ * Produce actionable frontmatter diagnostics directly from file content.
+ *
+ * This duplicates the YAML parse that `parseFrontmatterOnly` already does,
+ * but it runs only on the rare failure path (solution dropped from index),
+ * so the overhead is acceptable in exchange for a human-readable error list.
+ */
+export function diagnoseFromRawContent(content) {
+    const trimmed = content.trimStart();
+    if (!trimmed.startsWith('---'))
+        return ['no YAML frontmatter (missing leading ---)'];
+    const endIdx = trimmed.indexOf('---', 3);
+    if (endIdx === -1)
+        return ['frontmatter not closed (missing trailing ---)'];
+    const raw = trimmed.slice(3, endIdx);
+    if (raw.length > 5000)
+        return ['frontmatter too large (>5000 chars — YAML bomb guard)'];
+    let parsed;
+    try {
+        parsed = yaml.load(raw, { schema: yaml.JSON_SCHEMA });
+    }
+    catch (e) {
+        return [`YAML parse error: ${e instanceof Error ? e.message : String(e)}`];
+    }
+    return diagnoseFrontmatter(parsed);
+}
+/**
+ * Append one quarantine entry for `filePath`. Deduped by path within the
+ * current file: if the latest entry for this path already matches the
+ * current errors, skip the append.
+ *
+ * Storage: one JSONL line per quarantine event. Readers use only the
+ * latest line per path.
+ */
+export function recordQuarantine(filePath, errors) {
+    try {
+        fs.mkdirSync(STATE_DIR, { recursive: true });
+        if (dedupeHit(filePath, errors))
+            return;
+        const entry = {
+            path: filePath,
+            at: new Date().toISOString(),
+            errors,
+        };
+        fs.appendFileSync(SOLUTION_QUARANTINE_PATH, JSON.stringify(entry) + '\n');
+    }
+    catch (e) {
+        log.debug(`quarantine write failed: ${e instanceof Error ? e.message : String(e)}`);
+    }
+}
+function dedupeHit(filePath, errors) {
+    if (!fs.existsSync(SOLUTION_QUARANTINE_PATH))
+        return false;
+    try {
+        const text = fs.readFileSync(SOLUTION_QUARANTINE_PATH, 'utf-8');
+        const lines = text.split('\n').filter(Boolean);
+        for (let i = lines.length - 1; i >= 0; i--) {
+            let prev;
+            try {
+                prev = JSON.parse(lines[i]);
+            }
+            catch {
+                continue;
+            }
+            if (prev.path !== filePath)
+                continue;
+            if (sameErrors(prev.errors, errors))
+                return true;
+            return false;
+        }
+    }
+    catch { /* ignore */ }
+    return false;
+}
+function sameErrors(a, b) {
+    if (a.length !== b.length)
+        return false;
+    const sa = [...a].sort();
+    const sb = [...b].sort();
+    for (let i = 0; i < sa.length; i++)
+        if (sa[i] !== sb[i])
+            return false;
+    return true;
+}
+/**
+ * Read the latest quarantine state: one entry per path, keyed to the most
+ * recent append. Entries whose file no longer exists are dropped.
+ */
+export function listQuarantined() {
+    if (!fs.existsSync(SOLUTION_QUARANTINE_PATH))
+        return [];
+    let text;
+    try {
+        text = fs.readFileSync(SOLUTION_QUARANTINE_PATH, 'utf-8');
+    }
+    catch {
+        return [];
+    }
+    const byPath = new Map();
+    for (const line of text.split('\n')) {
+        if (!line)
+            continue;
+        try {
+            const entry = JSON.parse(line);
+            byPath.set(entry.path, entry);
+        }
+        catch { /* skip bad line */ }
+    }
+    const result = [];
+    for (const entry of byPath.values()) {
+        try {
+            if (fs.existsSync(entry.path))
+                result.push(entry);
+        }
+        catch { /* skip */ }
+    }
+    return result;
+}
+/**
+ * Clear quarantine entries for files that now parse correctly or no longer
+ * exist. Intended to be called after `forgen learn fix-up` or a manual edit.
+ */
+export function pruneQuarantine() {
+    if (!fs.existsSync(SOLUTION_QUARANTINE_PATH))
+        return { removed: 0, kept: 0 };
+    // Read raw entries without listQuarantined's existsSync filter so we can
+    // count deleted files as removed rather than silently dropping them.
+    const byPath = new Map();
+    try {
+        const text = fs.readFileSync(SOLUTION_QUARANTINE_PATH, 'utf-8');
+        for (const line of text.split('\n')) {
+            if (!line)
+                continue;
+            try {
+                const entry = JSON.parse(line);
+                byPath.set(entry.path, entry);
+            }
+            catch { /* skip bad line */ }
+        }
+    }
+    catch { /* empty */ }
+    const stillBad = [];
+    let removed = 0;
+    for (const entry of byPath.values()) {
+        let content;
+        try {
+            content = fs.readFileSync(entry.path, 'utf-8');
+        }
+        catch {
+            removed++;
+            continue;
+        }
+        const errors = diagnoseFromRawContent(content);
+        if (errors.length === 0) {
+            removed++;
+            continue;
+        }
+        stillBad.push({ ...entry, errors });
+    }
+    const dir = path.dirname(SOLUTION_QUARANTINE_PATH);
+    fs.mkdirSync(dir, { recursive: true });
+    const text = stillBad.map((e) => JSON.stringify(e)).join('\n') + (stillBad.length ? '\n' : '');
+    fs.writeFileSync(SOLUTION_QUARANTINE_PATH, text);
+    return { removed, kept: stillBad.length };
+}

package/dist/engine/solution-weakness.d.ts

@@ -0,0 +1,45 @@
+export interface UnderServedTag {
+    tag: string;
+    correction_mentions: number;
+    best_matching_champion: string | null;
+    best_fitness: number;
+}
+export interface ConflictCluster {
+    shared_tags: string[];
+    champion: {
+        name: string;
+        fitness: number;
+    };
+    underperform: {
+        name: string;
+        fitness: number;
+    };
+}
+export interface DeadCorner {
+    solution: string;
+    unique_tags: string[];
+    injected: number;
+}
+export interface VolatileSolution {
+    solution: string;
+    accept_rate_window_a: number;
+    accept_rate_window_b: number;
+    delta: number;
+}
+export interface WeaknessReport {
+    generated_at: string;
+    population: {
+        total: number;
+        champion: number;
+        active: number;
+        underperform: number;
+        draft: number;
+    };
+    under_served_tags: UnderServedTag[];
+    conflict_clusters: ConflictCluster[];
+    dead_corners: DeadCorner[];
+    volatile: VolatileSolution[];
+}
+export declare function buildWeaknessReport(solutionsDir?: string): WeaknessReport;
+export declare function saveWeaknessReport(report: WeaknessReport): string;
+export declare function latestWeaknessReport(): WeaknessReport | null;