@pugi/cli 0.1.0-beta.5 → 0.1.0-beta.51

package/dist/core/smoke/orchestrator.js

@@ -0,0 +1,194 @@
+/**
+ * Smoke orchestrator — glues the scenario parser, the headless
+ * subprocess driver, and the runner into a single "load → run → report"
+ * pipeline. The CLI surface (`pugi smoke`) and the standalone script
+ * (`scripts/run-scenarios.ts`) both call into this module so the two
+ * entry points share one code path.
+ *
+ * Phase 1 boundary — `runSmoke` is responsible for:
+ *
+ *   1. Discovering scenario files under `scenariosDir` (glob match on
+ *      `*.scenario.txt`).
+ *   2. Parsing each file via `parseScenario`. Parse errors are surfaced
+ *      via the report but do not stop the run — every scenario gets a
+ *      chance to fail with a clean diagnostic.
+ *   3. Driving each scenario through the headless executor (the
+ *      executor is injected so tests can swap it for a deterministic
+ *      stub; production wires `runHeadlessScenario` from
+ *      `headless-driver.ts`).
+ *   4. Filtering by `--filter <pattern>` (compiles to fnmatch-lite).
+ *   5. Computing pass/fail/summary numbers.
+ *
+ * The orchestrator is intentionally synchronous (apart from the
+ * per-scenario `await`) — running scenarios in parallel is a Phase 2
+ * concern. The corpus is small and sequential output is easier to read.
+ */
+import { readdirSync, readFileSync, statSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { parseScenario, } from './scenario-parser.js';
+import { runScenario, } from './runner.js';
+/**
+ * Top-level smoke entry. Returns the report so the CLI can pretty-print
+ * it AND set `process.exitCode` deterministically.
+ */
+export async function runSmoke(opts) {
+    const log = opts.log ?? noopLog;
+    const now = opts.now ?? Date.now;
+    const allScenarios = loadScenariosFromDir(opts.scenariosDir);
+    const visible = opts.filter && opts.filter.length > 0
+        ? filterByPattern(allScenarios, opts.filter)
+        : allScenarios;
+    const results = [];
+    let passed = 0;
+    let failed = 0;
+    for (const item of visible) {
+        log(`pugi smoke: running ${item.scenario.id}`);
+        if (item.parseErrors.length > 0) {
+            results.push({
+                id: item.scenario.id,
+                filePath: item.scenario.filePath,
+                status: 'parse-error',
+                durationMs: 0,
+                assertionCount: 0,
+                failures: [],
+                parseErrors: item.parseErrors,
+            });
+            failed += 1;
+            continue;
+        }
+        let envelopes = [];
+        let workspaceRoot = '.';
+        try {
+            const out = await opts.executor(item.scenario);
+            envelopes = out.envelopes;
+            workspaceRoot = out.workspaceRoot;
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            results.push({
+                id: item.scenario.id,
+                filePath: item.scenario.filePath,
+                status: 'executor-error',
+                durationMs: 0,
+                assertionCount: 0,
+                failures: [],
+                executorError: message,
+            });
+            failed += 1;
+            continue;
+        }
+        const result = runScenario({
+            scenario: item.scenario,
+            envelopes,
+            workspaceRoot,
+            now,
+        });
+        results.push({
+            id: result.id,
+            filePath: item.scenario.filePath,
+            status: result.passed ? 'passed' : 'failed',
+            durationMs: result.durationMs,
+            assertionCount: result.assertionCount,
+            failures: result.failures,
+        });
+        if (result.passed)
+            passed += 1;
+        else
+            failed += 1;
+    }
+    const total = visible.length;
+    const skipped = allScenarios.length - visible.length;
+    const exitCode = failed === 0 ? 0 : 1;
+    return { total, passed, failed, skipped, results, exitCode };
+}
+/**
+ * Walk `dir` for `*.scenario.txt` files (non-recursive). Returns each
+ * file's parsed scenario + collected parse errors so the orchestrator
+ * can surface malformed files as failed runs rather than skipping them.
+ */
+export function loadScenariosFromDir(dir) {
+    let names = [];
+    try {
+        names = readdirSync(dir);
+    }
+    catch {
+        return [];
+    }
+    const out = [];
+    for (const name of names) {
+        if (!name.endsWith('.scenario.txt'))
+            continue;
+        const filePath = resolve(dir, name);
+        let stat;
+        try {
+            stat = statSync(filePath);
+        }
+        catch {
+            continue;
+        }
+        if (!stat.isFile())
+            continue;
+        const body = readFileSync(filePath, 'utf8');
+        const parsed = parseScenario(filePath, body);
+        if (parsed.scenario) {
+            out.push({ scenario: parsed.scenario, parseErrors: parsed.errors });
+        }
+    }
+    // Stable sort by id so report output is deterministic across
+    // filesystems with different readdir order.
+    out.sort((a, b) => a.scenario.id.localeCompare(b.scenario.id));
+    return out;
+}
+/**
+ * Render a SmokeReport into a multi-line human-readable string. Kept
+ * separate from `runSmoke` so the CLI can pick its own format (text vs
+ * JSON). The default text format mirrors `node:test`'s tap-lite output:
+ *
+ *   ok 1 - identity (12ms)
+ *   not ok 2 - codegen-create-file (8ms)
+ *     line 5: EXPECT failed — no envelope matched ...
+ *
+ *   pugi smoke: 1 passed, 1 failed
+ */
+export function renderReportText(report) {
+    const lines = [];
+    for (let i = 0; i < report.results.length; i += 1) {
+        const r = report.results[i];
+        if (!r)
+            continue;
+        const ordinal = i + 1;
+        if (r.status === 'passed') {
+            lines.push(`ok ${ordinal} - ${r.id} (${r.durationMs}ms)`);
+        }
+        else if (r.status === 'failed') {
+            lines.push(`not ok ${ordinal} - ${r.id} (${r.durationMs}ms)`);
+            for (const f of r.failures) {
+                lines.push(`  line ${f.line}: ${f.message}`);
+            }
+        }
+        else if (r.status === 'parse-error') {
+            lines.push(`not ok ${ordinal} - ${r.id} (parse error)`);
+            for (const e of r.parseErrors ?? []) {
+                lines.push(`  line ${e.line}: ${e.message}`);
+            }
+        }
+        else if (r.status === 'executor-error') {
+            lines.push(`not ok ${ordinal} - ${r.id} (executor error)`);
+            lines.push(`  ${r.executorError ?? 'unknown executor failure'}`);
+        }
+    }
+    lines.push('');
+    const skippedSuffix = report.skipped > 0 ? `, ${report.skipped} skipped` : '';
+    lines.push(`pugi smoke: ${report.passed} passed, ${report.failed} failed${skippedSuffix}`);
+    return lines.join('\n');
+}
+function filterByPattern(scenarios, pattern) {
+    if (!pattern.includes('*')) {
+        return scenarios.filter((s) => s.scenario.id.includes(pattern));
+    }
+    const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, '\\$&');
+    const re = new RegExp(`^${escaped.replace(/\*/g, '.*')}$`);
+    return scenarios.filter((s) => re.test(s.scenario.id));
+}
+function noopLog(_line) { }
+//# sourceMappingURL=orchestrator.js.map

package/dist/core/smoke/runner.js

@@ -0,0 +1,238 @@
+/**
+ * Scenario runner for the Pugi MCP test harness (BIG TRACK 10 Phase 1,
+ * 2026-05-27).
+ *
+ * Drives a parsed scenario against a stream of `HeadlessEnvelope`
+ * objects (the same shape `pugi --headless` emits on stdout) and a
+ * filesystem checker for `EXPECT_FILE` directives. The runner is
+ * deliberately decoupled from the subprocess spawn so the spec file
+ * can inject deterministic envelope arrays without spawning a real
+ * `pugi` binary — that strategy keeps the test cycle under 200ms while
+ * still exercising the matching semantics every CI run depends on.
+ *
+ * Matching semantics (the rules the corpus authors care about):
+ *
+ *   - `EXPECT:` after a `>` user-input line scans envelopes that
+ *     arrived AFTER that user-input. The cursor resets on each new
+ *     `>`. If no envelope satisfies the pattern, the assertion fails.
+ *
+ *   - `EXPECT_NOT:` runs the inverse — passes if NO envelope in the
+ *     post-`>` window satisfies the pattern. A negative assertion that
+ *     fires on every input line gives the operator a clean signal when
+ *     a forbidden phrase ("Mira") shows up.
+ *
+ *   - `EXPECT_FILE:` runs once at the END of the scenario, against the
+ *     final filesystem snapshot. The runner does not race the
+ *     subprocess — by the time we evaluate file assertions the
+ *     headless process has exited (or been terminated).
+ *
+ * Result shape mirrors `node:test` style: top-level pass/fail plus an
+ * array of per-assertion records so the CLI can print a grouped
+ * summary. Each failure carries the originating line number so the
+ * operator can jump straight to the scenario source.
+ */
+import { existsSync, readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+/**
+ * Run the assertions in `scenario` against the given envelope stream
+ * and filesystem snapshot. Pure function — no I/O outside the filesystem
+ * stat that `EXPECT_FILE` performs, and even that is gated by a step
+ * actually existing.
+ */
+export function runScenario(inputs) {
+    const now = inputs.now ?? Date.now;
+    const startedAt = now();
+    const failures = [];
+    let assertionCount = 0;
+    // Group steps into runs anchored by `>` user-input lines. Each run
+    // owns the EXPECT/EXPECT_NOT assertions that follow it until the
+    // next `>`. EXPECT_FILE is collected globally and evaluated after
+    // every user-input run is processed.
+    const runs = [];
+    const fileChecks = [];
+    for (const step of inputs.scenario.steps) {
+        if (step.kind === 'user-input') {
+            runs.push({ userStep: step, expects: [] });
+            continue;
+        }
+        if (step.kind === 'expect') {
+            // Assertions that appear BEFORE any `>` attach to a synthetic
+            // pre-run so the matching pass still sees them. Rare in practice,
+            // but the parser allows it and the runner should not silently
+            // drop them.
+            if (runs.length === 0)
+                runs.push({ userStep: null, expects: [] });
+            const current = runs[runs.length - 1];
+            if (current)
+                current.expects.push(step);
+            continue;
+        }
+        if (step.kind === 'expect-file') {
+            fileChecks.push(step);
+        }
+    }
+    // Build a per-run envelope window. We walk the envelope stream once,
+    // assigning each envelope to the most recent user-turn we have seen.
+    // The first `user-turn` envelope after a `>` is the marker for that
+    // run; assertions match within the slice up to (but not including)
+    // the NEXT `user-turn` envelope.
+    const userTurnIndices = [];
+    for (let i = 0; i < inputs.envelopes.length; i += 1) {
+        const env = inputs.envelopes[i];
+        if (env && env.kind === 'user-turn')
+            userTurnIndices.push(i);
+    }
+    for (let runIdx = 0; runIdx < runs.length; runIdx += 1) {
+        const run = runs[runIdx];
+        if (!run)
+            continue;
+        let start = 0;
+        let end = inputs.envelopes.length;
+        if (run.userStep && userTurnIndices[runIdx] !== undefined) {
+            start = (userTurnIndices[runIdx] ?? 0) + 1;
+            const nextUserTurn = userTurnIndices[runIdx + 1];
+            end = nextUserTurn ?? inputs.envelopes.length;
+        }
+        const window = inputs.envelopes.slice(start, end);
+        for (const expectation of run.expects) {
+            assertionCount += 1;
+            const matched = window.some((env) => matchesEnvelope(env, expectation.pattern));
+            if (expectation.polarity === 'positive' && !matched) {
+                failures.push({
+                    line: expectation.line,
+                    message: `EXPECT failed — no envelope matched ${describePattern(expectation.pattern)}`,
+                });
+            }
+            else if (expectation.polarity === 'negative' && matched) {
+                failures.push({
+                    line: expectation.line,
+                    message: `EXPECT_NOT failed — envelope matched ${describePattern(expectation.pattern)}`,
+                });
+            }
+        }
+    }
+    for (const check of fileChecks) {
+        assertionCount += 1;
+        const absolute = resolve(inputs.workspaceRoot, check.file);
+        if (!existsSync(absolute)) {
+            failures.push({
+                line: check.line,
+                message: `EXPECT_FILE failed — ${check.file} does not exist`,
+            });
+            continue;
+        }
+        if (check.content !== undefined) {
+            const body = readFileSync(absolute, 'utf8');
+            if (!body.includes(check.content)) {
+                failures.push({
+                    line: check.line,
+                    message: `EXPECT_FILE failed — ${check.file} does not contain "${check.content}"`,
+                });
+            }
+        }
+    }
+    const durationMs = Math.max(0, now() - startedAt);
+    return {
+        id: inputs.scenario.id,
+        passed: failures.length === 0,
+        failures,
+        durationMs,
+        assertionCount,
+    };
+}
+/**
+ * Filter parsed scenarios by a simple glob-ish substring matcher.
+ * `*` matches any run of characters; otherwise we fall back to plain
+ * substring containment so `pugi smoke --filter identity` works as
+ * the operator expects. The matcher is intentionally NOT a full RegExp
+ * (no anchors, no character classes) because scenarios are addressed
+ * by short ids — a `--filter "id*"` form is the maximum complexity
+ * the corpus needs.
+ */
+export function filterScenarios(scenarios, pattern) {
+    if (!pattern || pattern.length === 0)
+        return scenarios;
+    const matcher = compileFilterPattern(pattern);
+    return scenarios.filter((s) => matcher(s.id));
+}
+function compileFilterPattern(pattern) {
+    if (!pattern.includes('*')) {
+        return (id) => id.includes(pattern);
+    }
+    // Escape RegExp metacharacters except `*`, then translate `*` to
+    // `.*`. This is the dropbox/glob "fnmatch lite" approach — predictable
+    // and small.
+    const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, '\\$&');
+    const re = new RegExp(`^${escaped.replace(/\*/g, '.*')}$`);
+    return (id) => re.test(id);
+}
+/**
+ * Decide whether a single envelope satisfies a pattern. Exported for
+ * tests that want to probe the matching logic without building a full
+ * scenario object.
+ */
+export function matchesEnvelope(env, pattern) {
+    if (pattern.kind === 'persona-turn-contains') {
+        if (env.kind !== 'persona-turn')
+            return false;
+        return pattern.substrings.some((s) => env.body.includes(s));
+    }
+    if (pattern.kind === 'tool-call') {
+        if (env.kind !== 'tool-call')
+            return false;
+        // The body is JSON. Tool calls that don't parse as JSON cannot
+        // match — surface a clean fail instead of crashing.
+        let parsed;
+        try {
+            parsed = JSON.parse(env.body);
+        }
+        catch {
+            return false;
+        }
+        if (!isRecord(parsed))
+            return false;
+        if (pattern.tool !== undefined && parsed['tool'] !== pattern.tool) {
+            return false;
+        }
+        if (pattern.argsSubset !== undefined) {
+            const args = parsed['args'];
+            if (!isRecord(args))
+                return false;
+            for (const [k, v] of Object.entries(pattern.argsSubset)) {
+                if (String(args[k]) !== v)
+                    return false;
+            }
+        }
+        return true;
+    }
+    if (pattern.kind === 'envelope-kind') {
+        return env.kind === pattern.envelopeKind;
+    }
+    return false;
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function describePattern(pattern) {
+    if (pattern.kind === 'persona-turn-contains') {
+        return `persona-turn containing one of [${pattern.substrings
+            .map((s) => `"${s}"`)
+            .join(', ')}]`;
+    }
+    if (pattern.kind === 'tool-call') {
+        const parts = [];
+        if (pattern.tool)
+            parts.push(`tool=${pattern.tool}`);
+        if (pattern.argsSubset) {
+            for (const [k, v] of Object.entries(pattern.argsSubset)) {
+                parts.push(`${k}=${v}`);
+            }
+        }
+        return `tool-call ${parts.join(' ')}`.trim();
+    }
+    if (pattern.kind === 'envelope-kind') {
+        return `envelope kind=${pattern.envelopeKind}`;
+    }
+    return 'unknown pattern';
+}
+//# sourceMappingURL=runner.js.map