@pugi/cli 0.1.0-beta.10 → 0.1.0-beta.101

package/dist/core/eval/v1/task-loader.js

@@ -0,0 +1,191 @@
+/**
+ * Task loader for pugi-eval-v1.
+ *
+ * Reads `<root>/eval/v1/tasks/<NN>-<slug>.task.yml`, parses via the
+ * embedded minimal YAML parser, validates via Zod, then asserts the
+ * filename matches the parsed `id` so a refactor cannot silently
+ * decouple the basename from the schema id.
+ *
+ * The loader also exposes `loadTaskManifest` which is the spec-side
+ * gate that pins task content via sha256 and refuses mismatches (the
+ * frozen-benchmark invariant from backlog #120).
+ */
+import { createHash } from 'node:crypto';
+import { readdirSync, readFileSync } from 'node:fs';
+import { basename, join, resolve } from 'node:path';
+import { z } from 'zod';
+import { parseTaskYaml } from './yaml-parser.js';
+const TASK_ID_RE = /^\d{2}-[a-z][a-z0-9-]*$/;
+const TASK_FILENAME_RE = /^(\d{2}-[a-z][a-z0-9-]*)\.task\.yml$/;
+const verificationSchema = z.discriminatedUnion('kind', [
+    z
+        .object({ kind: z.literal('file_exists'), path: z.string().min(1) })
+        .strict(),
+    z
+        .object({
+        kind: z.literal('file_contains'),
+        path: z.string().min(1),
+        pattern: z.string().min(1),
+        mode: z.enum(['literal', 'regex']).optional(),
+    })
+        .strict(),
+    z
+        .object({
+        kind: z.literal('output_contains'),
+        pattern: z.string().min(1),
+        mode: z.enum(['literal', 'regex']).optional(),
+    })
+        .strict(),
+    z
+        .object({
+        kind: z.literal('command_exit_code'),
+        command: z.string().min(1),
+        expectedExitCode: z.number().int(),
+        timeoutMs: z.number().int().positive().optional(),
+    })
+        .strict(),
+]);
+const taskSchema = z
+    .object({
+    id: z.string().regex(TASK_ID_RE, 'id must match <NN>-<slug>'),
+    difficulty: z.enum(['simple', 'medium', 'hard']),
+    intensity: z.enum(['quick', 'standard', 'deep', 'marathon']),
+    command: z.enum(['code', 'fix', 'explain', 'plan', 'build']),
+    brief: z.string().min(1),
+    fixture: z.record(z.string(), z.string()).optional(),
+    verification: z.array(verificationSchema).min(1),
+    maxTokens: z.number().int().positive(),
+    maxTurns: z.number().int().positive(),
+    timeoutMs: z.number().int().positive(),
+})
+    .strict();
+/**
+ * Default tasks directory relative to the @pugi/cli workspace root.
+ * Tests inject a different directory; production resolves it through
+ * `defaultTasksDir`.
+ */
+export function defaultTasksDir(packageRoot) {
+    return resolve(packageRoot, 'eval', 'v1', 'tasks');
+}
+export function defaultManifestPath(packageRoot) {
+    return resolve(packageRoot, 'eval', 'v1', 'manifest.json');
+}
+export function defaultLedgerPath(packageRoot) {
+    return resolve(packageRoot, 'eval', 'v1', 'results.tsv');
+}
+export function listTaskFiles(tasksDir) {
+    let entries;
+    try {
+        entries = readdirSync(tasksDir);
+    }
+    catch (err) {
+        throw new Error(`eval-v1 tasks directory not found at ${tasksDir}: ${err.message}`);
+    }
+    const files = entries.filter((e) => TASK_FILENAME_RE.test(e)).sort();
+    return files.map((f) => join(tasksDir, f));
+}
+export function loadTaskFile(path) {
+    const raw = readFileSync(path, 'utf8');
+    const filenameMatch = TASK_FILENAME_RE.exec(basename(path));
+    if (!filenameMatch) {
+        throw new Error(`eval-v1: filename ${basename(path)} does not match <NN>-<slug>.task.yml`);
+    }
+    const expectedId = filenameMatch[1];
+    let parsed;
+    try {
+        parsed = parseTaskYaml(raw);
+    }
+    catch (err) {
+        throw new Error(`eval-v1 task ${basename(path)} failed YAML parse: ${err.message}`);
+    }
+    const result = taskSchema.safeParse(parsed);
+    if (!result.success) {
+        throw new Error(`eval-v1 task ${basename(path)} failed schema validation: ${result.error.message}`);
+    }
+    if (result.data.id !== expectedId) {
+        throw new Error(`eval-v1 task ${basename(path)} id field ${result.data.id} does not match filename ${expectedId}`);
+    }
+    return { path, raw, spec: result.data };
+}
+export function loadAllTasks(tasksDir) {
+    const files = listTaskFiles(tasksDir);
+    const loaded = files.map((f) => loadTaskFile(f));
+    // Detect duplicate ids that survived (cannot happen given filename
+    // regex but defensive against future refactors).
+    const seen = new Set();
+    for (const entry of loaded) {
+        if (seen.has(entry.spec.id)) {
+            throw new Error(`eval-v1: duplicate task id ${entry.spec.id}`);
+        }
+        seen.add(entry.spec.id);
+    }
+    return loaded;
+}
+export function manifestEntryFor(path, raw) {
+    const filenameMatch = TASK_FILENAME_RE.exec(basename(path));
+    if (!filenameMatch) {
+        throw new Error(`cannot derive manifest entry: filename ${basename(path)} does not match`);
+    }
+    const id = filenameMatch[1];
+    const sha = createHash('sha256').update(raw).digest('hex');
+    return { id, sha256: sha, byteLength: Buffer.byteLength(raw, 'utf8') };
+}
+export function readManifest(manifestPath) {
+    const raw = readFileSync(manifestPath, 'utf8');
+    const parsed = JSON.parse(raw);
+    const schema = z
+        .object({
+        schemaVersion: z.literal(1),
+        generatedAt: z.string(),
+        entries: z
+            .array(z
+            .object({
+            id: z.string().regex(TASK_ID_RE),
+            sha256: z.string().regex(/^[0-9a-f]{64}$/),
+            byteLength: z.number().int().nonnegative(),
+        })
+            .strict())
+            .min(1),
+    })
+        .strict();
+    const result = schema.safeParse(parsed);
+    if (!result.success) {
+        throw new Error(`eval-v1 manifest ${manifestPath} invalid: ${result.error.message}`);
+    }
+    return result.data;
+}
+/**
+ * Compare the on-disk task files against the committed manifest. Used
+ * by the meta-spec to enforce the frozen-benchmark invariant.
+ */
+export function diffManifest(tasks, manifest) {
+    const reasons = [];
+    const computed = new Map();
+    for (const t of tasks) {
+        const entry = manifestEntryFor(t.path, t.raw);
+        computed.set(entry.id, entry);
+    }
+    const declared = new Map();
+    for (const e of manifest.entries)
+        declared.set(e.id, e);
+    for (const [id, entry] of computed) {
+        const decl = declared.get(id);
+        if (!decl) {
+            reasons.push(`task ${id} present on disk but missing from manifest`);
+            continue;
+        }
+        if (decl.sha256 !== entry.sha256) {
+            reasons.push(`task ${id} sha256 mismatch (disk=${entry.sha256.slice(0, 12)} manifest=${decl.sha256.slice(0, 12)})`);
+        }
+        if (decl.byteLength !== entry.byteLength) {
+            reasons.push(`task ${id} byteLength mismatch (disk=${entry.byteLength} manifest=${decl.byteLength})`);
+        }
+    }
+    for (const [id] of declared) {
+        if (!computed.has(id)) {
+            reasons.push(`task ${id} declared in manifest but missing on disk`);
+        }
+    }
+    return { ok: reasons.length === 0, reasons };
+}
+//# sourceMappingURL=task-loader.js.map

package/dist/core/eval/v1/types.js

@@ -0,0 +1,14 @@
+/**
+ * pugi-eval-v1 type definitions (backlog #120,  Reviewer foundation).
+ *
+ * Frozen benchmark harness types. The shapes here are stable: every
+ * field added later must preserve backward compatibility with the v1
+ * `results.tsv` ledger columns and the v1 task YAML schema. Breaking
+ * changes ship as `eval-v2`.
+ *
+ * Why types live in `core/eval/v1/` and not next to the CLI command:
+ * the meta-spec, ledger, scoring, and verifier all consume them. CLI
+ * command modules stay thin wrappers per the project convention.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/core/eval/v1/verifier.js

@@ -0,0 +1,176 @@
+/**
+ * Verification check executor for pugi-eval-v1.
+ *
+ * Each check kind has a pure implementation. The runner calls
+ * `runVerifications` after the CLI subprocess exits, passing the
+ * post-run workspace root + the final stdout text the engine emitted.
+ *
+ * Path safety: `file_exists` and `file_contains` resolve the path
+ * relative to the workspace root and refuse anything that escapes the
+ * root (`..` traversal, absolute paths, symlinks). A task spec that
+ * tries to peek outside the workspace fails the check rather than
+ * leaking host state.
+ */
+import { spawnSync } from 'node:child_process';
+import { existsSync, readFileSync, realpathSync, statSync } from 'node:fs';
+import { isAbsolute, relative, resolve } from 'node:path';
+const DEFAULT_COMMAND_TIMEOUT_MS = 60_000;
+function withinRoot(rootReal, candidateReal) {
+    if (candidateReal === rootReal)
+        return true;
+    const rel = relative(rootReal, candidateReal);
+    return rel !== '' && !rel.startsWith('..') && !isAbsolute(rel);
+}
+/**
+ * Resolve `workspaceRoot` to its realpath once so symlinked tmp dirs
+ * (macOS `/tmp` -> `/private/tmp`) compare correctly against any
+ * resolved child path. Falls back к the raw root when the directory
+ * does not yet exist (defensive - production callers always create
+ * the tmp dir before invoking).
+ */
+function realRoot(workspaceRoot) {
+    try {
+        return realpathSync(workspaceRoot);
+    }
+    catch {
+        return workspaceRoot;
+    }
+}
+function resolveWorkspacePath(workspaceRoot, path) {
+    if (isAbsolute(path)) {
+        return { ok: false, reason: `absolute paths refused (${path})` };
+    }
+    if (path.split(/[\\/]/).includes('..')) {
+        return { ok: false, reason: `path traversal refused (${path})` };
+    }
+    const root = realRoot(workspaceRoot);
+    const absolute = resolve(root, path);
+    if (!withinRoot(root, absolute)) {
+        return { ok: false, reason: `path escapes workspace root (${path})` };
+    }
+    // realpath check: if the file exists, the realpath must also be
+    // inside the workspace root - symlinks that point outside are a
+    // refusal even when the entry itself is inside.
+    if (existsSync(absolute)) {
+        try {
+            const real = realpathSync(absolute);
+            if (!withinRoot(root, real)) {
+                return {
+                    ok: false,
+                    reason: `symlink target escapes workspace root (${path})`,
+                };
+            }
+        }
+        catch {
+            // Stat failure during realpath: surface as missing rather than
+            // throwing - the file_exists check will record the failure.
+        }
+    }
+    return { ok: true, absolute };
+}
+function matches(haystack, pattern, mode) {
+    if (mode === 'regex') {
+        let re;
+        try {
+            re = new RegExp(pattern);
+        }
+        catch (err) {
+            return {
+                ok: false,
+                reason: `invalid regex ${pattern}: ${err.message}`,
+            };
+        }
+        return re.test(haystack)
+            ? { ok: true, reason: '' }
+            : { ok: false, reason: `regex ${pattern} did not match` };
+    }
+    return haystack.includes(pattern)
+        ? { ok: true, reason: '' }
+        : { ok: false, reason: `literal ${JSON.stringify(pattern)} not found` };
+}
+export function runVerification(check, ctx) {
+    switch (check.kind) {
+        case 'file_exists': {
+            const r = resolveWorkspacePath(ctx.workspaceRoot, check.path);
+            if (!r.ok)
+                return { kind: check.kind, passed: false, detail: r.reason };
+            try {
+                const stat = statSync(r.absolute);
+                if (!stat.isFile()) {
+                    return {
+                        kind: check.kind,
+                        passed: false,
+                        detail: `${check.path} exists but is not a regular file`,
+                    };
+                }
+                return { kind: check.kind, passed: true, detail: '' };
+            }
+            catch {
+                return {
+                    kind: check.kind,
+                    passed: false,
+                    detail: `${check.path} not found`,
+                };
+            }
+        }
+        case 'file_contains': {
+            const r = resolveWorkspacePath(ctx.workspaceRoot, check.path);
+            if (!r.ok)
+                return { kind: check.kind, passed: false, detail: r.reason };
+            let body;
+            try {
+                body = readFileSync(r.absolute, 'utf8');
+            }
+            catch {
+                return {
+                    kind: check.kind,
+                    passed: false,
+                    detail: `${check.path} not readable`,
+                };
+            }
+            const m = matches(body, check.pattern, check.mode);
+            return {
+                kind: check.kind,
+                passed: m.ok,
+                detail: m.ok ? '' : `${check.path}: ${m.reason}`,
+            };
+        }
+        case 'output_contains': {
+            const m = matches(ctx.finalText, check.pattern, check.mode);
+            return {
+                kind: check.kind,
+                passed: m.ok,
+                detail: m.ok ? '' : `final output: ${m.reason}`,
+            };
+        }
+        case 'command_exit_code': {
+            const timeout = check.timeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS;
+            const result = spawnSync('bash', ['-lc', check.command], {
+                cwd: ctx.workspaceRoot,
+                timeout,
+                encoding: 'utf8',
+                env: { ...process.env, CI: '1' },
+            });
+            if (result.error) {
+                return {
+                    kind: check.kind,
+                    passed: false,
+                    detail: `${check.command}: ${result.error.message}`,
+                };
+            }
+            const exit = result.status ?? -1;
+            if (exit !== check.expectedExitCode) {
+                return {
+                    kind: check.kind,
+                    passed: false,
+                    detail: `${check.command}: exit ${exit} expected ${check.expectedExitCode}`,
+                };
+            }
+            return { kind: check.kind, passed: true, detail: '' };
+        }
+    }
+}
+export function runVerifications(checks, ctx) {
+    return checks.map((c) => runVerification(c, ctx));
+}
+//# sourceMappingURL=verifier.js.map

package/dist/core/eval/v1/yaml-parser.js

@@ -0,0 +1,250 @@
+/**
+ * Minimal YAML parser for pugi-eval-v1 task files.
+ *
+ * Same rationale as `core/recipes/schema.ts`: pulling in `js-yaml`
+ * is ~70 KB of installed weight for a schema we control. The shape
+ * here is intentionally narrow:
+ *
+ *  - Top-level scalars (string / number / boolean).
+ *  - Block sequences of scalars (e.g. plain string lists).
+ *  - Block sequences of maps (verification entries).
+ *  - Nested block maps (fixture file map).
+ *  - Multi-line block scalars via `|` (literal) and `>` (folded).
+ *  - `#` line comments.
+ *
+ * Unsupported YAML features: flow collections, anchors, tags,
+ * directives, document separators. The parser throws on anything
+ * outside the supported subset so a malformed task file fails fast
+ * with a clear line number.
+ *
+ * The parsed value is `unknown`; downstream loader code applies a Zod
+ * schema to narrow it. The parser deliberately does NOT do schema
+ * validation - it only enforces YAML well-formedness.
+ */
+export class TaskYamlParseError extends Error {
+    line;
+    constructor(message, line) {
+        super(`task YAML parse error at line ${line}: ${message}`);
+        this.line = line;
+        this.name = 'TaskYamlParseError';
+    }
+}
+/**
+ * Tokenise the source into non-blank, non-comment lines + their
+ * indentation. Tab characters are rejected (YAML spec) so the indent
+ * count is unambiguous.
+ */
+function tokenise(source) {
+    const physical = source.split(/\r?\n/);
+    const out = [];
+    for (let i = 0; i < physical.length; i += 1) {
+        const raw = physical[i] ?? '';
+        const lineNo = i + 1;
+        if (raw.includes('\t')) {
+            throw new TaskYamlParseError('tab characters are not allowed; use spaces for indentation', lineNo);
+        }
+        const stripped = raw.replace(/\s*#.*$/, '');
+        if (stripped.trim() === '')
+            continue;
+        const indent = stripped.length - stripped.replace(/^ +/, '').length;
+        out.push({ lineNo, indent, body: stripped.slice(indent) });
+    }
+    return out;
+}
+/**
+ * Convert a literal scalar (after the `:` or after a `-` marker) into
+ * a JS value. Recognises `true`/`false`, integers, floats, single-
+ * and double-quoted strings; everything else is returned verbatim as
+ * an unquoted string.
+ */
+function coerceScalar(raw, lineNo) {
+    const trimmed = raw.trim();
+    if (trimmed === '')
+        return '';
+    if (trimmed === 'true')
+        return true;
+    if (trimmed === 'false')
+        return false;
+    if (trimmed === 'null' || trimmed === '~')
+        return '';
+    if (/^-?\d+$/.test(trimmed))
+        return Number(trimmed);
+    if (/^-?\d+\.\d+$/.test(trimmed))
+        return Number(trimmed);
+    const doubleQuoted = /^"((?:\\.|[^"\\])*)"$/.exec(trimmed);
+    if (doubleQuoted) {
+        // Decode common escape sequences. This subset matches what the
+        // task spec needs (newlines inside regex patterns, escaped
+        // backslashes).
+        return doubleQuoted[1].replace(/\\(.)/g, (_, c) => {
+            if (c === 'n')
+                return '\n';
+            if (c === 't')
+                return '\t';
+            if (c === 'r')
+                return '\r';
+            if (c === '"')
+                return '"';
+            if (c === '\\')
+                return '\\';
+            throw new TaskYamlParseError(`unknown escape \\${c} in double-quoted string`, lineNo);
+        });
+    }
+    const singleQuoted = /^'((?:''|[^'])*)'$/.exec(trimmed);
+    if (singleQuoted) {
+        return singleQuoted[1].replace(/''/g, "'");
+    }
+    return trimmed;
+}
+/**
+ * Block scalar reader for `|` (literal, preserves newlines) and `>`
+ * (folded, newlines collapse to spaces). Joins continuation lines
+ * whose indent is strictly greater than the parent map's indent.
+ */
+function readBlockScalar(marker, parentIndent, lines, cursor) {
+    const collected = [];
+    let baseIndent = -1;
+    while (cursor.i < lines.length) {
+        const next = lines[cursor.i];
+        if (next.indent <= parentIndent)
+            break;
+        if (baseIndent < 0)
+            baseIndent = next.indent;
+        const slice = ' '.repeat(Math.max(0, next.indent - baseIndent)) + next.body;
+        collected.push(slice);
+        cursor.i += 1;
+    }
+    if (marker === '|')
+        return collected.join('\n');
+    return collected.join(' ');
+}
+/**
+ * Recursively parse a block beginning at `cursor.i` whose body lives
+ * strictly deeper than `parentIndent`. The return value is either a
+ * map (object) or a sequence (array); the caller decides which based
+ * on the first child line.
+ */
+function parseBlock(lines, cursor, parentIndent) {
+    if (cursor.i >= lines.length)
+        return {};
+    const first = lines[cursor.i];
+    if (first.indent <= parentIndent)
+        return {};
+    const blockIndent = first.indent;
+    const isSequence = first.body.startsWith('- ') || first.body === '-';
+    if (isSequence) {
+        const arr = [];
+        while (cursor.i < lines.length) {
+            const line = lines[cursor.i];
+            if (line.indent < blockIndent)
+                break;
+            if (line.indent > blockIndent) {
+                throw new TaskYamlParseError(`unexpected indent ${line.indent} in sequence (expected ${blockIndent})`, line.lineNo);
+            }
+            if (!line.body.startsWith('-')) {
+                throw new TaskYamlParseError(`expected sequence marker '-' but found ${line.body.slice(0, 10)}`, line.lineNo);
+            }
+            const after = line.body.replace(/^-\s*/, '');
+            cursor.i += 1;
+            if (after === '') {
+                arr.push(parseBlock(lines, cursor, blockIndent));
+                continue;
+            }
+            // Inline `- key: value` first entry; treat the rest of the line
+            // as a single-key map entry, then continue collecting deeper
+            // siblings into the same map.
+            const colonIdx = after.indexOf(':');
+            if (colonIdx === -1) {
+                arr.push(coerceScalar(after, line.lineNo));
+                continue;
+            }
+            const key = after.slice(0, colonIdx).trim();
+            const valRaw = after.slice(colonIdx + 1).trim();
+            const entry = {};
+            if (valRaw === '' || valRaw === '|' || valRaw === '>') {
+                if (valRaw === '|' || valRaw === '>') {
+                    entry[key] = readBlockScalar(valRaw, blockIndent, lines, cursor);
+                }
+                else {
+                    entry[key] = parseBlock(lines, cursor, blockIndent);
+                }
+            }
+            else {
+                entry[key] = coerceScalar(valRaw, line.lineNo);
+            }
+            // Continue absorbing deeper siblings of this synthetic map.
+            while (cursor.i < lines.length) {
+                const sibling = lines[cursor.i];
+                // Sibling map keys live at the same indent as the inline
+                // `key: value` text (which is at `blockIndent + 2` relative to
+                // the `-` marker). Anything at blockIndent or shallower
+                // belongs to the parent.
+                const childIndent = blockIndent + 2;
+                if (sibling.indent < childIndent)
+                    break;
+                if (sibling.indent > childIndent) {
+                    throw new TaskYamlParseError(`unexpected indent ${sibling.indent} (expected ${childIndent})`, sibling.lineNo);
+                }
+                const sibColon = sibling.body.indexOf(':');
+                if (sibColon === -1) {
+                    throw new TaskYamlParseError(`expected key:value in map but found ${sibling.body.slice(0, 20)}`, sibling.lineNo);
+                }
+                const sibKey = sibling.body.slice(0, sibColon).trim();
+                const sibVal = sibling.body.slice(sibColon + 1).trim();
+                cursor.i += 1;
+                if (sibVal === '' || sibVal === '|' || sibVal === '>') {
+                    if (sibVal === '|' || sibVal === '>') {
+                        entry[sibKey] = readBlockScalar(sibVal, childIndent, lines, cursor);
+                    }
+                    else {
+                        entry[sibKey] = parseBlock(lines, cursor, childIndent);
+                    }
+                }
+                else {
+                    entry[sibKey] = coerceScalar(sibVal, sibling.lineNo);
+                }
+            }
+            arr.push(entry);
+        }
+        return arr;
+    }
+    const map = {};
+    while (cursor.i < lines.length) {
+        const line = lines[cursor.i];
+        if (line.indent < blockIndent)
+            break;
+        if (line.indent > blockIndent) {
+            throw new TaskYamlParseError(`unexpected indent ${line.indent} (expected ${blockIndent})`, line.lineNo);
+        }
+        const colonIdx = line.body.indexOf(':');
+        if (colonIdx === -1) {
+            throw new TaskYamlParseError(`expected key:value but found ${line.body.slice(0, 20)}`, line.lineNo);
+        }
+        const key = line.body.slice(0, colonIdx).trim();
+        const valRaw = line.body.slice(colonIdx + 1).trim();
+        cursor.i += 1;
+        if (valRaw === '') {
+            map[key] = parseBlock(lines, cursor, blockIndent);
+            continue;
+        }
+        if (valRaw === '|' || valRaw === '>') {
+            map[key] = readBlockScalar(valRaw, blockIndent, lines, cursor);
+            continue;
+        }
+        map[key] = coerceScalar(valRaw, line.lineNo);
+    }
+    return map;
+}
+export function parseTaskYaml(source) {
+    const lines = tokenise(source);
+    if (lines.length === 0)
+        return {};
+    const cursor = { i: 0 };
+    const result = parseBlock(lines, cursor, -1);
+    if (cursor.i !== lines.length) {
+        const stray = lines[cursor.i];
+        throw new TaskYamlParseError(`stray content after document body`, stray.lineNo);
+    }
+    return result;
+}
+//# sourceMappingURL=yaml-parser.js.map