@pugi/cli 0.1.0-beta.98 → 1.0.0-alpha.1

package/dist/core/smoke/headless-driver.js

@@ -1,174 +0,0 @@
-/**
- * Headless subprocess driver for the smoke harness (Phase 1).
- *
- * Spawns `pugi --headless`, feeds the scenario's user-input lines one
- * at a time, and collects every JSON envelope written to stdout. The
- * driver is the bridge between the in-process scenario parser and the
- * real CLI binary — running scenarios via subprocess is what lets
- * `pugi smoke` validate the AS-PUBLISHED CLI, not just the in-process
- * code path.
- *
- * Phase 1 invariants:
- *
- *  - One scenario = one subprocess. We do NOT reuse processes across
- *    scenarios; each run starts in a fresh temp workspace so file
- *    assertions are deterministic.
- *
- *  - Stdin is fed line-by-line on a timer (default 200ms gap). We
- *    could fire all lines at once, but spacing them out gives the
- *    engine a chance to emit `persona-turn` envelopes between user
- *    turns. Mirrors real-operator typing cadence — feeds a more
- *    realistic test signal than a burst write.
- *
- *  - Hard timeout per scenario (default 30s). The driver kills the
- *    subprocess on timeout and surfaces a clean executor-error so the
- *    report still shows the row instead of hanging CI.
- *
- *  - Stderr is captured but NOT parsed. We forward it to the optional
- *    `onStderr` sink so the operator can see what the headless CLI
- *    wrote to stderr (banners, MCP load notices) — handy for
- *    debugging a failing scenario locally.
- */
-import { spawn } from 'node:child_process';
-import { mkdtempSync, rmSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import { resolve } from 'node:path';
-/**
- * Run a parsed scenario through the headless CLI. Returns the captured
- * envelope stream + the workspace root the process ran in so the
- * runner can resolve `EXPECT_FILE` paths.
- */
-export async function runHeadlessScenario(scenario, opts = {}) {
-    const pugiBin = opts.pugiBin ?? 'pugi';
-    const extraArgs = opts.extraArgs ?? [];
-    const timeoutMs = opts.timeoutMs ?? 30_000;
-    const lineGapMs = opts.lineGapMs ?? 200;
-    const onStderr = opts.onStderr ?? noopChunk;
-    const spawner = opts.spawner ?? defaultSpawner;
-    const workspaceRoot = mkdtempSync(resolve(tmpdir(), 'pugi-smoke-'));
-    let cleanedUp = false;
-    const args = ['--headless', ...extraArgs];
-    const child = spawner(pugiBin, args, {
-        cwd: workspaceRoot,
-        env: { ...process.env, PUGI_HEADLESS: '1' },
-    });
-    const envelopes = [];
-    let stdoutBuffer = '';
-    child.stdout.setEncoding('utf8');
-    child.stdout.on('data', (chunk) => {
-        stdoutBuffer += chunk;
-        let newlineIndex = stdoutBuffer.indexOf('\n');
-        while (newlineIndex !== -1) {
-            const line = stdoutBuffer.slice(0, newlineIndex);
-            stdoutBuffer = stdoutBuffer.slice(newlineIndex + 1);
-            const parsed = parseEnvelopeLine(line);
-            if (parsed)
-                envelopes.push(parsed);
-            newlineIndex = stdoutBuffer.indexOf('\n');
-        }
-    });
-    child.stderr.setEncoding('utf8');
-    child.stderr.on('data', (chunk) => onStderr(chunk));
-    const userInputs = scenario.steps
-        .filter((s) => s.kind === 'user-input')
-        .map((s) => s.body);
-    // Feed inputs serially with a small gap. We don't await the engine's
-    // response — Phase 1 evaluates the WHOLE envelope stream after the
-    // process closes, so the gap is purely to give the engine room to
-    // emit between turns. A future phase can swap this for a barrier
-    // ("wait for `persona-turn` before sending the next `user-turn`")
-    // once envelope ordering is sealed.
-    const feedPromise = (async () => {
-        for (const line of userInputs) {
-            child.stdin.write(`${line}\n`);
-            await delay(lineGapMs);
-        }
-        child.stdin.end();
-    })();
-    const timeoutHandle = setTimeout(() => {
-        child.kill('SIGTERM');
-    }, timeoutMs);
-    // Don't keep node alive purely for the timeout — the spawned child
-    // already holds the loop via its IPC pipe.
-    timeoutHandle.unref?.();
-    await new Promise((resolvePromise) => {
-        child.on('close', () => resolvePromise());
-    });
-    clearTimeout(timeoutHandle);
-    // Flush any tail bytes that did not end with a newline.
-    if (stdoutBuffer.length > 0) {
-        const parsed = parseEnvelopeLine(stdoutBuffer);
-        if (parsed)
-            envelopes.push(parsed);
-        stdoutBuffer = '';
-    }
-    await feedPromise.catch(() => {
-        /* feed promise rejects when the child closes early — ignore */
-    });
-    // Cleanup the temp workspace lazily — the orchestrator may want to
-    // inspect it after the run. Callers that need a fresh dir per scenario
-    // should pass a custom workspaceRoot in a future phase; for now the
-    // runner resolves EXPECT_FILE paths against this directory and the
-    // OS reaps the tmp tree on reboot.
-    // (Explicit cleanup helper exposed for tests that want determinism.)
-    void cleanedUp;
-    return { envelopes, workspaceRoot };
-}
-/**
- * Tear down a workspace created by `runHeadlessScenario`. Exposed
- * separately because the orchestrator wants the directory to survive
- * the run for EXPECT_FILE evaluation.
- */
-export function cleanupWorkspace(workspaceRoot) {
-    try {
-        rmSync(workspaceRoot, { recursive: true, force: true });
-    }
-    catch {
-        /* best-effort */
-    }
-}
-/**
- * Parse one line of headless stdout into an envelope. Returns null
- * when the line is empty or unparseable — non-fatal so the runner can
- * tolerate stray banner output.
- */
-export function parseEnvelopeLine(raw) {
-    const line = raw.trim();
-    if (line.length === 0)
-        return null;
-    if (!line.startsWith('{'))
-        return null;
-    try {
-        const parsed = JSON.parse(line);
-        if (typeof parsed['kind'] !== 'string')
-            return null;
-        if (typeof parsed['body'] !== 'string')
-            return null;
-        const ts = typeof parsed['ts'] === 'number'
-            ? parsed['ts']
-            : Number(parsed['ts'] ?? Date.now());
-        return {
-            kind: parsed['kind'],
-            body: parsed['body'],
-            ts: Number.isFinite(ts) ? ts : Date.now(),
-        };
-    }
-    catch {
-        return null;
-    }
-}
-function defaultSpawner(bin, args, options) {
-    return spawn(bin, args, {
-        cwd: options.cwd,
-        env: options.env,
-        stdio: ['pipe', 'pipe', 'pipe'],
-    });
-}
-function delay(ms) {
-    return new Promise((resolve) => {
-        const handle = setTimeout(resolve, ms);
-        handle.unref?.();
-    });
-}
-function noopChunk(_chunk) { }
-//# sourceMappingURL=headless-driver.js.map

package/dist/core/smoke/orchestrator.js

@@ -1,194 +0,0 @@
-/**
- * Smoke orchestrator — glues the scenario parser, the headless
- * subprocess driver, and the runner into a single "load → run → report"
- * pipeline. The CLI surface (`pugi smoke`) and the standalone script
- * (`scripts/run-scenarios.ts`) both call into this module so the two
- * entry points share one code path.
- *
- * Phase 1 boundary — `runSmoke` is responsible for:
- *
- *  1. Discovering scenario files under `scenariosDir` (glob match on
- *     `*.scenario.txt`).
- *  2. Parsing each file via `parseScenario`. Parse errors are surfaced
- *     via the report but do not stop the run — every scenario gets a
- *     chance to fail with a clean diagnostic.
- *  3. Driving each scenario through the headless executor (the
- *     executor is injected so tests can swap it for a deterministic
- *     stub; production wires `runHeadlessScenario` from
- *     `headless-driver.ts`).
- *  4. Filtering by `--filter <pattern>` (compiles to fnmatch-lite).
- *  5. Computing pass/fail/summary numbers.
- *
- * The orchestrator is intentionally synchronous (apart from the
- * per-scenario `await`) — running scenarios in parallel is a Phase 2
- * concern. The corpus is small and sequential output is easier to read.
- */
-import { readdirSync, readFileSync, statSync } from 'node:fs';
-import { resolve } from 'node:path';
-import { parseScenario, } from './scenario-parser.js';
-import { runScenario, } from './runner.js';
-/**
- * Top-level smoke entry. Returns the report so the CLI can pretty-print
- * it AND set `process.exitCode` deterministically.
- */
-export async function runSmoke(opts) {
-    const log = opts.log ?? noopLog;
-    const now = opts.now ?? Date.now;
-    const allScenarios = loadScenariosFromDir(opts.scenariosDir);
-    const visible = opts.filter && opts.filter.length > 0
-        ? filterByPattern(allScenarios, opts.filter)
-        : allScenarios;
-    const results = [];
-    let passed = 0;
-    let failed = 0;
-    for (const item of visible) {
-        log(`pugi smoke: running ${item.scenario.id}`);
-        if (item.parseErrors.length > 0) {
-            results.push({
-                id: item.scenario.id,
-                filePath: item.scenario.filePath,
-                status: 'parse-error',
-                durationMs: 0,
-                assertionCount: 0,
-                failures: [],
-                parseErrors: item.parseErrors,
-            });
-            failed += 1;
-            continue;
-        }
-        let envelopes = [];
-        let workspaceRoot = '.';
-        try {
-            const out = await opts.executor(item.scenario);
-            envelopes = out.envelopes;
-            workspaceRoot = out.workspaceRoot;
-        }
-        catch (error) {
-            const message = error instanceof Error ? error.message : String(error);
-            results.push({
-                id: item.scenario.id,
-                filePath: item.scenario.filePath,
-                status: 'executor-error',
-                durationMs: 0,
-                assertionCount: 0,
-                failures: [],
-                executorError: message,
-            });
-            failed += 1;
-            continue;
-        }
-        const result = runScenario({
-            scenario: item.scenario,
-            envelopes,
-            workspaceRoot,
-            now,
-        });
-        results.push({
-            id: result.id,
-            filePath: item.scenario.filePath,
-            status: result.passed ? 'passed' : 'failed',
-            durationMs: result.durationMs,
-            assertionCount: result.assertionCount,
-            failures: result.failures,
-        });
-        if (result.passed)
-            passed += 1;
-        else
-            failed += 1;
-    }
-    const total = visible.length;
-    const skipped = allScenarios.length - visible.length;
-    const exitCode = failed === 0 ? 0 : 1;
-    return { total, passed, failed, skipped, results, exitCode };
-}
-/**
- * Walk `dir` for `*.scenario.txt` files (non-recursive). Returns each
- * file's parsed scenario + collected parse errors so the orchestrator
- * can surface malformed files as failed runs rather than skipping them.
- */
-export function loadScenariosFromDir(dir) {
-    let names = [];
-    try {
-        names = readdirSync(dir);
-    }
-    catch {
-        return [];
-    }
-    const out = [];
-    for (const name of names) {
-        if (!name.endsWith('.scenario.txt'))
-            continue;
-        const filePath = resolve(dir, name);
-        let stat;
-        try {
-            stat = statSync(filePath);
-        }
-        catch {
-            continue;
-        }
-        if (!stat.isFile())
-            continue;
-        const body = readFileSync(filePath, 'utf8');
-        const parsed = parseScenario(filePath, body);
-        if (parsed.scenario) {
-            out.push({ scenario: parsed.scenario, parseErrors: parsed.errors });
-        }
-    }
-    // Stable sort by id so report output is deterministic across
-    // filesystems with different readdir order.
-    out.sort((a, b) => a.scenario.id.localeCompare(b.scenario.id));
-    return out;
-}
-/**
- * Render a SmokeReport into a multi-line human-readable string. Kept
- * separate from `runSmoke` so the CLI can pick its own format (text vs
- * JSON). The default text format mirrors `node:test`'s tap-lite output:
- *
- *  ok 1 - identity (12ms)
- *  not ok 2 - codegen-create-file (8ms)
- *    line 5: EXPECT failed — no envelope matched ...
- *
- *  pugi smoke: 1 passed, 1 failed
- */
-export function renderReportText(report) {
-    const lines = [];
-    for (let i = 0; i < report.results.length; i += 1) {
-        const r = report.results[i];
-        if (!r)
-            continue;
-        const ordinal = i + 1;
-        if (r.status === 'passed') {
-            lines.push(`ok ${ordinal} - ${r.id} (${r.durationMs}ms)`);
-        }
-        else if (r.status === 'failed') {
-            lines.push(`not ok ${ordinal} - ${r.id} (${r.durationMs}ms)`);
-            for (const f of r.failures) {
-                lines.push(` line ${f.line}: ${f.message}`);
-            }
-        }
-        else if (r.status === 'parse-error') {
-            lines.push(`not ok ${ordinal} - ${r.id} (parse error)`);
-            for (const e of r.parseErrors ?? []) {
-                lines.push(` line ${e.line}: ${e.message}`);
-            }
-        }
-        else if (r.status === 'executor-error') {
-            lines.push(`not ok ${ordinal} - ${r.id} (executor error)`);
-            lines.push(` ${r.executorError ?? 'unknown executor failure'}`);
-        }
-    }
-    lines.push('');
-    const skippedSuffix = report.skipped > 0 ? `, ${report.skipped} skipped` : '';
-    lines.push(`pugi smoke: ${report.passed} passed, ${report.failed} failed${skippedSuffix}`);
-    return lines.join('\n');
-}
-function filterByPattern(scenarios, pattern) {
-    if (!pattern.includes('*')) {
-        return scenarios.filter((s) => s.scenario.id.includes(pattern));
-    }
-    const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, '\\$&');
-    const re = new RegExp(`^${escaped.replace(/\*/g, '.*')}$`);
-    return scenarios.filter((s) => re.test(s.scenario.id));
-}
-function noopLog(_line) { }
-//# sourceMappingURL=orchestrator.js.map

package/dist/core/smoke/runner.js

@@ -1,238 +0,0 @@
-/**
- * Scenario runner for the Pugi MCP test harness (
- *).
- *
- * Drives a parsed scenario against a stream of `HeadlessEnvelope`
- * objects (the same shape `pugi --headless` emits on stdout) and a
- * filesystem checker for `EXPECT_FILE` directives. The runner is
- * deliberately decoupled from the subprocess spawn so the spec file
- * can inject deterministic envelope arrays without spawning a real
- * `pugi` binary — that strategy keeps the test cycle under 200ms while
- * still exercising the matching semantics every CI run depends on.
- *
- * Matching semantics (the rules the corpus authors care about):
- *
- *  - `EXPECT:` after a `>` user-input line scans envelopes that
- *    arrived AFTER that user-input. The cursor resets on each new
- *    `>`. If no envelope satisfies the pattern, the assertion fails.
- *
- *  - `EXPECT_NOT:` runs the inverse — passes if NO envelope in the
- *    post-`>` window satisfies the pattern. A negative assertion that
- *    fires on every input line gives the operator a clean signal when
- *    a forbidden phrase ("Pugi") shows up.
- *
- *  - `EXPECT_FILE:` runs once at the END of the scenario, against the
- *    final filesystem snapshot. The runner does not race the
- *    subprocess — by the time we evaluate file assertions the
- *    headless process has exited (or been terminated).
- *
- * Result shape mirrors `node:test` style: top-level pass/fail plus an
- * array of per-assertion records so the CLI can print a grouped
- * summary. Each failure carries the originating line number so the
- * operator can jump straight to the scenario source.
- */
-import { existsSync, readFileSync } from 'node:fs';
-import { resolve } from 'node:path';
-/**
- * Run the assertions in `scenario` against the given envelope stream
- * and filesystem snapshot. Pure function — no I/O outside the filesystem
- * stat that `EXPECT_FILE` performs, and even that is gated by a step
- * actually existing.
- */
-export function runScenario(inputs) {
-    const now = inputs.now ?? Date.now;
-    const startedAt = now();
-    const failures = [];
-    let assertionCount = 0;
-    // Group steps into runs anchored by `>` user-input lines. Each run
-    // owns the EXPECT/EXPECT_NOT assertions that follow it until the
-    // next `>`. EXPECT_FILE is collected globally and evaluated after
-    // every user-input run is processed.
-    const runs = [];
-    const fileChecks = [];
-    for (const step of inputs.scenario.steps) {
-        if (step.kind === 'user-input') {
-            runs.push({ userStep: step, expects: [] });
-            continue;
-        }
-        if (step.kind === 'expect') {
-            // Assertions that appear BEFORE any `>` attach to a synthetic
-            // pre-run so the matching pass still sees them. Rare in practice,
-            // but the parser allows it and the runner should not silently
-            // drop them.
-            if (runs.length === 0)
-                runs.push({ userStep: null, expects: [] });
-            const current = runs[runs.length - 1];
-            if (current)
-                current.expects.push(step);
-            continue;
-        }
-        if (step.kind === 'expect-file') {
-            fileChecks.push(step);
-        }
-    }
-    // Build a per-run envelope window. We walk the envelope stream once,
-    // assigning each envelope to the most recent user-turn we have seen.
-    // The first `user-turn` envelope after a `>` is the marker for that
-    // run; assertions match within the slice up to (but not including)
-    // the NEXT `user-turn` envelope.
-    const userTurnIndices = [];
-    for (let i = 0; i < inputs.envelopes.length; i += 1) {
-        const env = inputs.envelopes[i];
-        if (env && env.kind === 'user-turn')
-            userTurnIndices.push(i);
-    }
-    for (let runIdx = 0; runIdx < runs.length; runIdx += 1) {
-        const run = runs[runIdx];
-        if (!run)
-            continue;
-        let start = 0;
-        let end = inputs.envelopes.length;
-        if (run.userStep && userTurnIndices[runIdx] !== undefined) {
-            start = (userTurnIndices[runIdx] ?? 0) + 1;
-            const nextUserTurn = userTurnIndices[runIdx + 1];
-            end = nextUserTurn ?? inputs.envelopes.length;
-        }
-        const window = inputs.envelopes.slice(start, end);
-        for (const expectation of run.expects) {
-            assertionCount += 1;
-            const matched = window.some((env) => matchesEnvelope(env, expectation.pattern));
-            if (expectation.polarity === 'positive' && !matched) {
-                failures.push({
-                    line: expectation.line,
-                    message: `EXPECT failed — no envelope matched ${describePattern(expectation.pattern)}`,
-                });
-            }
-            else if (expectation.polarity === 'negative' && matched) {
-                failures.push({
-                    line: expectation.line,
-                    message: `EXPECT_NOT failed — envelope matched ${describePattern(expectation.pattern)}`,
-                });
-            }
-        }
-    }
-    for (const check of fileChecks) {
-        assertionCount += 1;
-        const absolute = resolve(inputs.workspaceRoot, check.file);
-        if (!existsSync(absolute)) {
-            failures.push({
-                line: check.line,
-                message: `EXPECT_FILE failed — ${check.file} does not exist`,
-            });
-            continue;
-        }
-        if (check.content !== undefined) {
-            const body = readFileSync(absolute, 'utf8');
-            if (!body.includes(check.content)) {
-                failures.push({
-                    line: check.line,
-                    message: `EXPECT_FILE failed — ${check.file} does not contain "${check.content}"`,
-                });
-            }
-        }
-    }
-    const durationMs = Math.max(0, now() - startedAt);
-    return {
-        id: inputs.scenario.id,
-        passed: failures.length === 0,
-        failures,
-        durationMs,
-        assertionCount,
-    };
-}
-/**
- * Filter parsed scenarios by a simple glob-ish substring matcher.
- * `*` matches any run of characters; otherwise we fall back to plain
- * substring containment so `pugi smoke --filter identity` works as
- * the operator expects. The matcher is intentionally NOT a full RegExp
- * (no anchors, no character classes) because scenarios are addressed
- * by short ids — a `--filter "id*"` form is the maximum complexity
- * the corpus needs.
- */
-export function filterScenarios(scenarios, pattern) {
-    if (!pattern || pattern.length === 0)
-        return scenarios;
-    const matcher = compileFilterPattern(pattern);
-    return scenarios.filter((s) => matcher(s.id));
-}
-function compileFilterPattern(pattern) {
-    if (!pattern.includes('*')) {
-        return (id) => id.includes(pattern);
-    }
-    // Escape RegExp metacharacters except `*`, then translate `*` to
-    // `.*`. This is the dropbox/glob "fnmatch lite" approach — predictable
-    // and small.
-    const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, '\\$&');
-    const re = new RegExp(`^${escaped.replace(/\*/g, '.*')}$`);
-    return (id) => re.test(id);
-}
-/**
- * Decide whether a single envelope satisfies a pattern. Exported for
- * tests that want to probe the matching logic without building a full
- * scenario object.
- */
-export function matchesEnvelope(env, pattern) {
-    if (pattern.kind === 'persona-turn-contains') {
-        if (env.kind !== 'persona-turn')
-            return false;
-        return pattern.substrings.some((s) => env.body.includes(s));
-    }
-    if (pattern.kind === 'tool-call') {
-        if (env.kind !== 'tool-call')
-            return false;
-        // The body is JSON. Tool calls that don't parse as JSON cannot
-        // match — surface a clean fail instead of crashing.
-        let parsed;
-        try {
-            parsed = JSON.parse(env.body);
-        }
-        catch {
-            return false;
-        }
-        if (!isRecord(parsed))
-            return false;
-        if (pattern.tool !== undefined && parsed['tool'] !== pattern.tool) {
-            return false;
-        }
-        if (pattern.argsSubset !== undefined) {
-            const args = parsed['args'];
-            if (!isRecord(args))
-                return false;
-            for (const [k, v] of Object.entries(pattern.argsSubset)) {
-                if (String(args[k]) !== v)
-                    return false;
-            }
-        }
-        return true;
-    }
-    if (pattern.kind === 'envelope-kind') {
-        return env.kind === pattern.envelopeKind;
-    }
-    return false;
-}
-function isRecord(value) {
-    return typeof value === 'object' && value !== null && !Array.isArray(value);
-}
-function describePattern(pattern) {
-    if (pattern.kind === 'persona-turn-contains') {
-        return `persona-turn containing one of [${pattern.substrings
-            .map((s) => `"${s}"`)
-            .join(', ')}]`;
-    }
-    if (pattern.kind === 'tool-call') {
-        const parts = [];
-        if (pattern.tool)
-            parts.push(`tool=${pattern.tool}`);
-        if (pattern.argsSubset) {
-            for (const [k, v] of Object.entries(pattern.argsSubset)) {
-                parts.push(`${k}=${v}`);
-            }
-        }
-        return `tool-call ${parts.join(' ')}`.trim();
-    }
-    if (pattern.kind === 'envelope-kind') {
-        return `envelope kind=${pattern.envelopeKind}`;
-    }
-    return 'unknown pattern';
-}
-//# sourceMappingURL=runner.js.map