@pugi/cli 0.1.0-beta.10 → 0.1.0-beta.101

package/dist/runtime/commands/eval-v1.js

@@ -0,0 +1,266 @@
+/**
+ * `pugi eval-v1` - run the frozen 20-task benchmark harness (#120).
+ *
+ * Surface:
+ *
+ *   pugi eval-v1 --list                       # list task ids + briefs
+ *   pugi eval-v1 --task 01-snake-html         # run one task
+ *   pugi eval-v1 --task 01,02,03              # run a subset
+ *   pugi eval-v1 --all                        # run the full 20
+ *   pugi eval-v1 --all --model qwen3-coder    # pin the engine model
+ *   pugi eval-v1 --all --json                 # JSON envelope output
+ *   pugi eval-v1 --all --no-ledger            # skip results.tsv append
+ *
+ * The handler stays thin: it resolves the tasks directory + pugi
+ * binary, calls `runHarness`, appends every result к the ledger
+ * (unless `--no-ledger`), and renders either a text table or a JSON
+ * envelope.
+ *
+ * Aggregate scoring (mean across all per-task scores) drives the CI
+ * regression gate in `.github/workflows/eval-v1.yml`. The gate is
+ * "aggregatePugiScore must not drop more than 10% vs the last week
+ * median" - the workflow computes the median itself from the
+ * committed ledger; the CLI's only job is к produce a fresh row.
+ */
+import { execFileSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { defaultLedgerPath, defaultTasksDir, loadAllTasks, } from '../../core/eval/v1/task-loader.js';
+import { aggregateScore } from '../../core/eval/v1/scoring.js';
+import { appendLedgerRows } from '../../core/eval/v1/ledger.js';
+import { runHarness } from '../../core/eval/v1/runner.js';
+function parseFlags(args) {
+    const out = {
+        list: false,
+        all: false,
+        task: [],
+        model: undefined,
+        ledger: true,
+        tasksDir: undefined,
+        ledgerPath: undefined,
+        help: false,
+    };
+    for (let i = 0; i < args.length; i += 1) {
+        const arg = args[i] ?? '';
+        if (arg === '--list')
+            out.list = true;
+        else if (arg === '--all')
+            out.all = true;
+        else if (arg === '--no-ledger')
+            out.ledger = false;
+        else if (arg === '--help' || arg === '-h')
+            out.help = true;
+        else if (arg === '--task') {
+            const next = args[i + 1];
+            if (!next || next.startsWith('--')) {
+                throw new Error('--task requires a comma-separated list of ids');
+            }
+            out.task.push(...next.split(',').map((s) => s.trim()).filter(Boolean));
+            i += 1;
+        }
+        else if (arg.startsWith('--task=')) {
+            out.task.push(...arg.slice('--task='.length).split(',').map((s) => s.trim()).filter(Boolean));
+        }
+        else if (arg === '--model') {
+            const next = args[i + 1];
+            if (!next)
+                throw new Error('--model requires a value');
+            out.model = next;
+            i += 1;
+        }
+        else if (arg.startsWith('--model=')) {
+            out.model = arg.slice('--model='.length);
+        }
+        else if (arg === '--tasks-dir') {
+            const next = args[i + 1];
+            if (!next)
+                throw new Error('--tasks-dir requires a path');
+            out.tasksDir = next;
+            i += 1;
+        }
+        else if (arg.startsWith('--tasks-dir=')) {
+            out.tasksDir = arg.slice('--tasks-dir='.length);
+        }
+        else if (arg === '--ledger') {
+            const next = args[i + 1];
+            if (!next)
+                throw new Error('--ledger requires a path');
+            out.ledgerPath = next;
+            i += 1;
+        }
+        else if (arg.startsWith('--ledger=')) {
+            out.ledgerPath = arg.slice('--ledger='.length);
+        }
+        else if (arg === '--json') {
+            // accepted at top-level; ignore here
+        }
+        else {
+            throw new Error(`unknown eval-v1 flag: ${arg}`);
+        }
+    }
+    return out;
+}
+function resolvePackageRoot(override) {
+    if (override)
+        return resolve(override);
+    // The compiled file lives at `dist/runtime/commands/eval-v1.js`;
+    // climb three directories to reach the package root in both dev
+    // (`src/runtime/commands/eval-v1.ts`) and dist builds.
+    const here = dirname(fileURLToPath(import.meta.url));
+    return resolve(here, '..', '..', '..');
+}
+function resolvePugiBin(override, packageRoot) {
+    if (override)
+        return resolve(override);
+    const local = resolve(packageRoot, 'bin', 'run.js');
+    if (existsSync(local))
+        return local;
+    // Fall back к PATH lookup; spawn will pick it up.
+    return 'pugi';
+}
+function captureGitSha(packageRoot) {
+    try {
+        const out = execFileSync('git', ['rev-parse', '--short', 'HEAD'], {
+            cwd: packageRoot,
+            encoding: 'utf8',
+            stdio: ['ignore', 'pipe', 'ignore'],
+        });
+        const trimmed = out.trim();
+        return trimmed === '' ? '(unknown)' : trimmed;
+    }
+    catch {
+        return '(unknown)';
+    }
+}
+function renderTable(report) {
+    const headers = ['id', 'difficulty', 'status', 'score', 'tokens', 'wall_ms', 'verifs'];
+    const rows = report.results.map((r) => [
+        r.taskId,
+        findDifficulty(r.taskId, report) ?? '?',
+        r.status,
+        r.pugiScore.toFixed(2),
+        String(r.tokensUsed),
+        String(r.wallClockMs),
+        `${r.verifications.filter((v) => v.passed).length}/${r.verifications.length}`,
+    ]);
+    const widths = headers.map((h, i) => Math.max(h.length, ...rows.map((r) => (r[i] ?? '').length)));
+    const renderRow = (cells) => cells.map((c, i) => c.padEnd(widths[i] ?? 0)).join('  ');
+    const lines = [renderRow(headers), renderRow(widths.map((w) => '-'.repeat(w)))];
+    for (const row of rows)
+        lines.push(renderRow(row));
+    lines.push('');
+    lines.push(`aggregate pugi_score: ${report.aggregatePugiScore.toFixed(2)} (mean of ${report.results.length})`);
+    lines.push(`model: ${report.model}`);
+    lines.push(`git_sha: ${report.gitSha}`);
+    return lines.join('\n');
+}
+const difficultyMap = new WeakMap();
+function findDifficulty(taskId, report) {
+    const cached = difficultyMap.get(report);
+    return cached?.get(taskId);
+}
+function indexDifficulty(report, specs) {
+    const map = new Map();
+    for (const s of specs)
+        map.set(s.id, s.difficulty);
+    difficultyMap.set(report, map);
+}
+export async function runEvalV1Command(ctx) {
+    let flags;
+    try {
+        flags = parseFlags(ctx.args);
+    }
+    catch (err) {
+        ctx.writeOutput({ command: 'eval-v1', status: 'invalid_args', reason: err.message }, `pugi eval-v1: ${err.message}`);
+        return 2;
+    }
+    if (flags.help) {
+        const help = [
+            'pugi eval-v1 - frozen 20-task benchmark harness (#120)',
+            '',
+            'usage:',
+            '  pugi eval-v1 --list',
+            '  pugi eval-v1 --task <id[,id...]>',
+            '  pugi eval-v1 --all [--model <slug>] [--no-ledger] [--json]',
+        ].join('\n');
+        ctx.writeOutput({ command: 'eval-v1', status: 'help', help }, help);
+        return 0;
+    }
+    const packageRoot = resolvePackageRoot(ctx.packageRoot);
+    const tasksDir = flags.tasksDir ?? defaultTasksDir(packageRoot);
+    const ledgerPath = flags.ledgerPath ?? defaultLedgerPath(packageRoot);
+    const log = ctx.log ?? ((line) => process.stderr.write(`${line}\n`));
+    let loaded;
+    try {
+        loaded = loadAllTasks(tasksDir);
+    }
+    catch (err) {
+        ctx.writeOutput({
+            command: 'eval-v1',
+            status: 'tasks_load_failed',
+            reason: err.message,
+        }, `pugi eval-v1: ${err.message}`);
+        return 2;
+    }
+    const specs = loaded.map((l) => l.spec);
+    if (flags.list) {
+        const indexLines = specs.map((s) => `${s.id}  [${s.command}/${s.difficulty}]  ${s.brief}`);
+        const out = indexLines.join('\n');
+        ctx.writeOutput({ command: 'eval-v1', status: 'listed', tasks: specs.map((s) => ({
+                id: s.id, command: s.command, difficulty: s.difficulty, brief: s.brief,
+            })) }, out);
+        return 0;
+    }
+    if (!flags.all && flags.task.length === 0) {
+        const reason = 'pass --all OR --task <id[,id...]> OR --list';
+        ctx.writeOutput({ command: 'eval-v1', status: 'invalid_args', reason }, `pugi eval-v1: ${reason}`);
+        return 2;
+    }
+    const only = flags.all ? undefined : flags.task;
+    if (only) {
+        const known = new Set(specs.map((s) => s.id));
+        const unknown = only.filter((id) => !known.has(id));
+        if (unknown.length) {
+            const reason = `unknown task id(s): ${unknown.join(', ')}`;
+            ctx.writeOutput({ command: 'eval-v1', status: 'invalid_args', reason }, `pugi eval-v1: ${reason}`);
+            return 2;
+        }
+    }
+    const pugiBin = resolvePugiBin(ctx.pugiBin, packageRoot);
+    const startedAt = new Date().toISOString();
+    const gitSha = captureGitSha(packageRoot);
+    const results = await runHarness({
+        specs,
+        options: {
+            pugiBin,
+            ...(flags.model !== undefined ? { model: flags.model } : {}),
+            ...(only !== undefined ? { only } : {}),
+            ...(ctx.runner ? { runner: ctx.runner } : {}),
+            ...(ctx.env !== undefined ? { env: ctx.env } : {}),
+            onTaskStart: (s) => log(`[eval-v1] ${s.id} starting (${s.command}/${s.difficulty})`),
+            onTaskFinish: (r) => log(`[eval-v1] ${r.taskId} ${r.status} score=${r.pugiScore.toFixed(2)} tokens=${r.tokensUsed} wall=${r.wallClockMs}ms`),
+        },
+    });
+    const report = {
+        schemaVersion: 1,
+        startedAt,
+        model: flags.model ?? '(default)',
+        results,
+        aggregatePugiScore: aggregateScore(results),
+        gitSha,
+    };
+    indexDifficulty(report, specs);
+    if (flags.ledger && results.length > 0) {
+        appendLedgerRows(ledgerPath, results.map((r) => ({
+            timestamp: startedAt,
+            gitSha,
+            model: report.model,
+            result: r,
+        })));
+    }
+    ctx.writeOutput(report, renderTable(report));
+    const anyFail = results.some((r) => r.status !== 'pass');
+    return anyFail ? 1 : 0;
+}
+//# sourceMappingURL=eval-v1.js.map

package/dist/runtime/commands/feedback.js

@@ -0,0 +1,184 @@
+/**
+ * `pugi feedback` + `/feedback` slash — .
+ *
+ * In-CLI feedback collector. Parity with the upstream tool's `/feedback`
+ * built-in. The operator never has to leave the terminal to file a
+ * bug / feature / general comment / praise. The wizard collects:
+ *
+ *  1. category  (bug / feature / general / praise)
+ *  2. rating    (1-5)
+ *  3. comment   (multi-line free text)
+ *  4. optional  redacted session context (last 5 turns)
+ *  5. confirm   (final y/n)
+ *
+ * # Module contract
+ *
+ *  - This file owns the WIRING from the CLI surface (TTY mount,
+ *    non-TTY JSON, slash dispatcher) to the queue + submitter
+ *    modules. The corpus + redactor + queue persistence live in
+ *    `core/feedback/{queue.ts,submitter.ts}`. The Ink prompt lives
+ *    in `tui/feedback-prompt.tsx`. Both have zero coupling to the
+ *    CLI dispatch surface.
+ *
+ *  - `runFeedbackCommand` is the single entry point. Both the top-
+ *    level `pugi feedback` handler в `runtime/cli.ts` AND the in-REPL
+ *    `/feedback` slash dispatcher call it. The function returns the
+ *    resolved `FeedbackRunResult` so the slash dispatcher can route
+ *    the outcome message to the REPL's system pane without re-prompting.
+ *
+ *  - Exit code is ALWAYS 0. Feedback is a brand surface — never a
+ *    gate. Failures land as result variants; the wrapper never
+ *    turns a network blip into a non-zero shell exit.
+ *
+ *  - The random-source-style test seam: the run helper accepts an
+ *    `interactive` flag that the spec sets to false, plus an injected
+ *    `draft` so unit tests can drive the submit + queue branches
+ *    without mounting Ink.
+ */
+import { existsSync, readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { enqueueFeedback, feedbackQueuePath, flushFeedbackQueue, } from '../../core/feedback/queue.js';
+import { feedbackSubmitUrl, redactSessionContext, submitFeedback, } from '../../core/feedback/submitter.js';
+/**
+ * Drive one feedback round. The function is async because of the
+ * submit round-trip, but everything else (queue write, redaction) is
+ * sync — no surprise concurrency.
+ */
+export async function runFeedbackCommand(ctx) {
+    if (ctx.draft == null) {
+        return { kind: 'cancelled' };
+    }
+    const envelope = {
+        category: ctx.draft.category,
+        rating: ctx.draft.rating,
+        comment: ctx.draft.comment,
+        ts: new Date().toISOString(),
+        cliVersion: ctx.cliVersion,
+        ...(ctx.tier ? { tier: ctx.tier } : {}),
+    };
+    if (ctx.draft.includeSessionContext && ctx.sessionContext) {
+        const sc = ctx.sessionContext();
+        if (sc)
+            envelope.sessionContext = sc;
+    }
+    let result;
+    try {
+        result = await ctx.submit(envelope);
+    }
+    catch (err) {
+        // Defensive: a thrown submitter (should not happen — the live
+        // submitter catches everything) is treated as transient so the
+        // envelope lands in the queue.
+        const reason = err instanceof Error ? err.message : String(err);
+        result = { kind: 'transient', reason };
+    }
+    if (result.kind === 'ok') {
+        return { kind: 'submitted', envelope, httpStatus: result.httpStatus };
+    }
+    if (result.kind === 'transient') {
+        const path = enqueueFeedback(envelope, ctx.cwd);
+        return { kind: 'queued', envelope, path, reason: result.reason };
+    }
+    // permanent — log + drop
+    return {
+        kind: 'dropped',
+        envelope,
+        reason: result.reason,
+        httpStatus: result.httpStatus,
+    };
+}
+/**
+ * Render one human-readable toast for the operator. Centralised so the
+ * top-level `pugi feedback` shell handler + the in-REPL `/feedback`
+ * slash dispatcher agree on the copy.
+ */
+export function renderFeedbackToast(result) {
+    switch (result.kind) {
+        case 'submitted':
+            return 'Feedback submitted. Thank you.';
+        case 'queued':
+            return `Feedback queued locally. Will sync on next online run. (${result.path})`;
+        case 'cancelled':
+            return 'Feedback cancelled. Nothing was sent.';
+        case 'dropped':
+            return `Feedback rejected by server (${result.httpStatus}): ${result.reason}. Not queued.`;
+        case 'noop':
+            return `No feedback collected: ${result.reason}`;
+    }
+}
+/**
+ * Background queue flush. Invoked silently on session start so any
+ * envelopes that landed during an offline run get drained when the
+ * operator next has connectivity. The function never throws — it
+ * returns the flush stats so the caller can log them at debug level.
+ */
+export async function flushFeedbackQueueSilently(cwd, config) {
+    // Short-circuit when the queue file does not exist. Avoids a
+    // pointless `fs.stat` round-trip on every cold session start.
+    if (!existsSync(feedbackQueuePath(cwd))) {
+        return { attempted: 0, succeeded: 0, failed: 0 };
+    }
+    const result = await flushFeedbackQueue(cwd, async (env) => {
+        const r = await submitFeedback(env, config);
+        if (r.kind === 'ok')
+            return true;
+        if (r.kind === 'permanent') {
+            // Permanent failures are "done" from the queue's POV — they
+            // would never resolve on retry. Drop them so the queue does
+            // not grow without bound.
+            return true;
+        }
+        return false;
+    });
+    return {
+        attempted: result.attempted,
+        succeeded: result.succeeded,
+        failed: result.failed,
+    };
+}
+/**
+ * Re-exports — the spec imports these via the command module so the
+ * dependency graph in the test stays single-rooted.
+ */
+export { feedbackQueuePath, feedbackSubmitUrl, redactSessionContext, submitFeedback, };
+/**
+ * Read the persona conversation log if present. Best-effort: returns
+ * an empty list when the file is missing or malformed. The CLI's REPL
+ * persists transcripts via the session module at a canonical relative
+ * path under `.pugi/sessions/`. The shell-level `pugi feedback` does
+ * not have access to a live session, so it tries to pick up the most
+ * recent persisted one for the `--with-context` path.
+ *
+ * Intentionally tolerant — feedback works even with no transcript.
+ */
+export function readMostRecentTranscript(cwd, options = {}) {
+    // The CLI may persist sessions in several places depending on the
+    // surface. We probe the conventional default; the spec drives the
+    // function via a fixture file instead of a live REPL.
+    const candidate = resolve(cwd, '.pugi', 'sessions', 'latest.jsonl');
+    if (!existsSync(candidate))
+        return [];
+    try {
+        const text = readFileSync(candidate, 'utf8');
+        const lines = text.split('\n').filter((l) => l.trim().length > 0);
+        const turns = [];
+        for (const line of lines) {
+            try {
+                const obj = JSON.parse(line);
+                if ((obj.role === 'user' || obj.role === 'assistant' || obj.role === 'system')
+                    && typeof obj.text === 'string') {
+                    turns.push({ role: obj.role, text: obj.text });
+                }
+            }
+            catch {
+                // skip malformed line
+            }
+        }
+        const cap = options.maxTurns ?? 5;
+        return turns.slice(-cap);
+    }
+    catch {
+        return [];
+    }
+}
+//# sourceMappingURL=feedback.js.map

package/dist/runtime/commands/hooks.js

@@ -0,0 +1,187 @@
+/**
+ * `pugi hooks` — operator surface for user-config hooks (MVP).
+ *
+ * Two subcommands ship in the MVP:
+ *
+ *  pugi hooks list        List configured hooks per event.
+ *  pugi hooks doctor      Validate the config and surface any
+ *                          parse / schema errors.
+ *
+ * Both accept `--json` for scripted callers. Argument grammar is
+ * intentionally narrow — no `add` / `remove` / `test` subcommands in
+ * the MVP. Operators hand-edit `~/.pugi/hooks-mvp.json` for now.
+ *
+ * Exit codes:
+ *  0 -> happy path (no hooks OR config valid).
+ *  1 -> config present but invalid (only `doctor` returns this).
+ *  2 -> unknown subcommand / argument error.
+ *
+ * Brand voice: ASCII only.
+ */
+import { ALL_HOOK_EVENTS_V2, defaultHooksMvpPath, loadHooksConfig, } from '../../core/hooks/index.js';
+function parseFlags(args) {
+    const rest = [];
+    const flags = { json: false };
+    for (const arg of args) {
+        if (arg === '--json') {
+            flags.json = true;
+            continue;
+        }
+        rest.push(arg);
+    }
+    return { rest, flags };
+}
+/**
+ * Top-level dispatcher for `pugi hooks <subcommand>`. Returns the
+ * intended process exit code. `cli.ts` is expected to set
+ * `process.exitCode = <return value>` so error states propagate to
+ * scripted callers without throwing.
+ */
+export async function runHooksCommand(args, ctx) {
+    const { rest, flags } = parseFlags(args);
+    const sub = rest[0];
+    if (!sub || sub === 'help' || sub === '--help') {
+        emitUsage(ctx, flags);
+        return sub ? 0 : 2;
+    }
+    if (sub === 'list') {
+        return runList(ctx, flags);
+    }
+    if (sub === 'doctor') {
+        return runDoctor(ctx, flags);
+    }
+    ctx.writeOutput({ ok: false, error: `unknown subcommand: ${sub}` }, `pugi hooks: unknown subcommand '${sub}'. Try 'pugi hooks --help'.`);
+    return 2;
+}
+function emitUsage(ctx, flags) {
+    const text = [
+        'pugi hooks — user-config lifecycle hooks (MVP).',
+        '',
+        'Subcommands:',
+        ' pugi hooks list       Show hooks configured per event.',
+        ' pugi hooks doctor     Validate ~/.pugi/hooks-mvp.json.',
+        '',
+        'Flags:',
+        ' --json                Emit a JSON envelope instead of human text.',
+        '',
+        'Config file:',
+        ' ~/.pugi/hooks-mvp.json',
+        '',
+        'Status:',
+        ' MVP — 2 events out of 8. Remaining events (PostToolUse,',
+        " UserPromptSubmit, Stop, SubagentStop, PreCompact, Notification)",
+        ' deferred to fast-follow PR.',
+    ].join('\n');
+    ctx.writeOutput({
+        ok: true,
+        command: 'hooks',
+        usage: text,
+    }, text);
+    if (flags.json) {
+        // The structured payload is already emitted by writeOutput when
+        // --json is on; nothing extra to do.
+    }
+}
+function runList(ctx, flags) {
+    let config;
+    try {
+        config = loadHooksConfig(ctx.configPath);
+    }
+    catch (error) {
+        const msg = error.message;
+        ctx.writeOutput({ ok: false, error: msg }, `pugi hooks list: ${msg}\nFix the config or remove the file. Run 'pugi hooks doctor' for details.`);
+        return 1;
+    }
+    const perEvent = {
+        SessionStart: [],
+        PreToolUse: [],
+        PostToolUse: [],
+        UserPromptSubmit: [],
+        Stop: [],
+        SubagentStop: [],
+        PreCompact: [],
+        Notification: [],
+        // PUGI-487 - worktree lifecycle events.
+        WorktreeCreate: [],
+        WorktreeRemove: [],
+    };
+    for (const event of ALL_HOOK_EVENTS_V2) {
+        perEvent[event] = config.list(event).map((entry) => ({
+            matcher: entry.matcher,
+            command: entry.command,
+            timeoutMs: entry.timeoutMs,
+            blocking: entry.blocking,
+        }));
+    }
+    const total = Object.values(perEvent).reduce((acc, list) => acc + list.length, 0);
+    const payload = {
+        ok: true,
+        configPath: config.configPath(),
+        total,
+        perEvent,
+    };
+    if (flags.json) {
+        ctx.writeOutput(payload, JSON.stringify(payload, null, 2));
+        return 0;
+    }
+    const lines = [];
+    lines.push(`pugi hooks (${total} configured)`);
+    lines.push(` config: ${config.configPath()}`);
+    if (total === 0) {
+        lines.push(' no hooks configured — create the file above to add one.');
+    }
+    else {
+        for (const event of ALL_HOOK_EVENTS_V2) {
+            const list = perEvent[event];
+            if (list.length === 0)
+                continue;
+            lines.push(` ${event}:`);
+            for (const entry of list) {
+                const tags = [];
+                if (entry.matcher)
+                    tags.push(`matcher=${entry.matcher}`);
+                if (entry.timeoutMs)
+                    tags.push(`timeoutMs=${entry.timeoutMs}`);
+                if (entry.blocking)
+                    tags.push('blocking');
+                const suffix = tags.length ? ` [${tags.join(', ')}]` : '';
+                lines.push(`   - ${entry.command}${suffix}`);
+            }
+        }
+    }
+    const text = lines.join('\n');
+    ctx.writeOutput(payload, text);
+    return 0;
+}
+function runDoctor(ctx, flags) {
+    const path = ctx.configPath ?? defaultHooksMvpPath();
+    try {
+        const config = loadHooksConfig(ctx.configPath);
+        const total = config.flatten().length;
+        const payload = {
+            ok: true,
+            configPath: config.configPath(),
+            total,
+            issues: [],
+        };
+        const text = total
+            ? `pugi hooks doctor: ${path} OK (${total} hooks).`
+            : `pugi hooks doctor: ${path} not present (no hooks configured).`;
+        ctx.writeOutput(payload, text);
+        return 0;
+    }
+    catch (error) {
+        const msg = error.message;
+        const payload = {
+            ok: false,
+            configPath: path,
+            error: msg,
+        };
+        const text = `pugi hooks doctor: ${msg}`;
+        ctx.writeOutput(payload, text);
+        return 1;
+    }
+    // flags.json is consumed by writeOutput in the host shell.
+    void flags;
+}
+//# sourceMappingURL=hooks.js.map