gaunt-sloth 2.0.0-alpha.2 → 2.0.0-alpha.21

package/dist/commands/evalCommand.js

@@ -0,0 +1,483 @@
+import { readdirSync, statSync } from 'node:fs';
+import { basename, extname, join, resolve } from 'node:path';
+import { initConfig, resolveIdentityProfileConfigPath, } from '@gaunt-sloth/core/config.js';
+import { getAskSystemPrompt } from '#src/commands/commandIntrospection.js';
+import { buildProductionRunCell, buildProductionRunConversation, } from '#src/commands/batchCommand.js';
+import { parseIntOption } from '#src/commands/cliOptionParsers.js';
+import { displayError, displaySuccess, displayWarning, } from '@gaunt-sloth/core/utils/consoleUtils.js';
+import { getProjectDir, setExitCode } from '@gaunt-sloth/core/utils/systemUtils.js';
+import { fileSafeLocalDate, getGslothFilePath, importExternalFile, readFileFromProjectDir, } from '@gaunt-sloth/core/utils/fileUtils.js';
+/** Split each collected `--reporter` value on `,`, flatten, trim, drop blanks, and de-duplicate
+ * (order-preserving), so `-r junit,text` and `-r junit -r text` both yield `['junit','text']`. */
+export function normalizeReporterNames(collected) {
+    const seen = new Set();
+    const out = [];
+    for (const raw of collected) {
+        for (const name of raw.split(',').map((n) => n.trim())) {
+            if (name && !seen.has(name)) {
+                seen.add(name);
+                out.push(name);
+            }
+        }
+    }
+    return out;
+}
+/**
+ * Build the `custom` reporter-factory map handed to `resolveReporters`. It always registers the
+ * BUNDLED JUnit reporter through the SAME public `custom` seam a user reporter uses (the point: it
+ * proves the plug-in contract from an out-of-core package), then overlays any config-declared
+ * reporters. A config reporter may reuse a built-in/bundled name — config wins on a collision, so it
+ * is added last.
+ *
+ * A config reporter's module path is resolved relative to the PROJECT dir (the same base
+ * `readFileFromProjectDir` uses) and imported via {@link importExternalFile} (which turns it into a
+ * `file://` URL and supports `.ts` through jiti). Its DEFAULT export must be an `EvalReporterFactory`
+ * (`() => EvalReporter`); a missing file, a failed import, or a non-function default export THROWS —
+ * caught by the command's outer try/catch → exit 2 (harness error). It is the user's own trusted
+ * config (already arbitrary JS), so nothing is sandboxed.
+ */
+async function buildCustomReporterFactories(config) {
+    const custom = {};
+    const { createJUnitReporter, JUNIT_REPORTER_NAME } = await import('@gaunt-sloth/eval-reporter-junit/index.js');
+    custom[JUNIT_REPORTER_NAME] = createJUnitReporter;
+    for (const [name, modulePath] of Object.entries(config.reporters ?? {})) {
+        const absPath = resolve(getProjectDir(), modulePath);
+        let mod;
+        try {
+            mod = await importExternalFile(absPath);
+        }
+        catch (error) {
+            throw new Error(`failed to load config reporter "${name}" from "${modulePath}" (${absPath}): ` +
+                (error instanceof Error ? error.message : String(error)));
+        }
+        const factory = mod.default;
+        if (typeof factory !== 'function') {
+            throw new Error(`config reporter "${name}" (${modulePath}) must export a default function ` +
+                `(an EvalReporterFactory: () => EvalReporter); got ${typeof factory}.`);
+        }
+        custom[name] = factory;
+    }
+    return custom;
+}
+/** Default output dir when `-o/--output` is omitted: a timestamped dir next to other gth reports —
+ * same convention as `defaultBatchOutputDir` in `batchCommand.ts`, `_EVAL` suffix instead. */
+export function defaultEvalOutputDir() {
+    return getGslothFilePath(`gth_${fileSafeLocalDate()}_EVAL`);
+}
+/** `basename(path)` with its extension removed — the per-suite output subdir name and the JUnit
+ * `<testsuites>` name. `eval/authz-matrix.yaml` → `authz-matrix`. */
+function suiteStem(path) {
+    const base = basename(path);
+    const ext = extname(base);
+    return ext ? base.slice(0, -ext.length) : base;
+}
+/**
+ * Deterministically resolve the CLI `<suites...>` args into the ordered list of suites to run
+ * (BATCH-19 Task B, Deliverable 1):
+ * - A **file** (or a path that doesn't exist on disk — its read is deferred so a missing suite
+ *   surfaces as that suite's own harness error, keeping the single-suite behavior byte-for-byte) →
+ *   that one suite, read by the exact CLI string given.
+ * - A **directory** → its **direct-child** `*.yaml`/`*.yml` files only (NON-recursive), sorted
+ *   lexicographically. An empty / no-match directory yields no suites and one harness `error` (the
+ *   caller folds it into the aggregate exit → 2) instead of throwing, so any GOOD sibling inputs
+ *   still run.
+ *
+ * The given arg order is preserved and each directory is expanded in place; suites are
+ * de-duplicated by **resolved absolute path** (the same file named twice, directly or via a
+ * directory, runs once).
+ */
+function resolveEvalSuiteInputs(inputs) {
+    const projectDir = getProjectDir();
+    const suites = [];
+    const errors = [];
+    const seen = new Set();
+    const push = (readPath, absPath) => {
+        if (seen.has(absPath))
+            return;
+        seen.add(absPath);
+        suites.push({ readPath, absPath });
+    };
+    for (const input of inputs) {
+        const abs = resolve(projectDir, input);
+        let isDirectory = false;
+        try {
+            isDirectory = statSync(abs).isDirectory();
+        }
+        catch {
+            // Non-existent path: treat as a file suite and let its read fail as that suite's own harness
+            // error (single-suite behavior preserved; a real file mock/read is keyed on the CLI string).
+            isDirectory = false;
+        }
+        if (!isDirectory) {
+            push(input, abs);
+            continue;
+        }
+        const yamlChildren = readdirSync(abs)
+            .filter((name) => {
+            const ext = extname(name).toLowerCase();
+            return ext === '.yaml' || ext === '.yml';
+        })
+            .sort();
+        if (yamlChildren.length === 0) {
+            errors.push(`eval suite directory "${input}" contains no .yaml/.yml files`);
+            continue;
+        }
+        for (const name of yamlChildren) {
+            const childAbs = resolve(abs, name);
+            push(childAbs, childAbs);
+        }
+    }
+    return { suites, errors };
+}
+/**
+ * Build the production {@link JudgeFn}: grades one case's answer with `judgeEvalCase` against
+ * `config.llm`. The caller chooses *which* config this is (BATCH-10 Task 2): by default it's the
+ * SUT's own config (judge shares the SUT model, the original behavior), but when a judge identity
+ * profile is resolved (`--judge <profile>` or the suite's `judge_profile`) the action passes a
+ * *separate* `initConfig({ …, identityProfile })` here, so the judge runs under its own model — a
+ * different model can catch blind spots the SUT model shares. This function stays agnostic to that
+ * choice; it just grades against whatever `config.llm` it's handed.
+ */
+function buildProductionJudge(config) {
+    return async (answer, rubric) => {
+        const { judgeEvalCase } = await import('@gaunt-sloth/batch/judge.js');
+        return judgeEvalCase(answer, rubric, config.llm);
+    };
+}
+/**
+ * Pure resolver for the judge identity profile — kept small and side-effect-free so it's unit
+ * testable in isolation. Precedence (highest first): CLI `--judge <profile>` > suite-level
+ * `judge_profile` > none. A blank/whitespace-only value at either level counts as absent (falls
+ * through to the next level). `undefined` means "no separate judge — grade with the SUT's config".
+ */
+export function resolveJudgeProfile(cliJudge, suiteJudgeProfile) {
+    const cli = cliJudge?.trim();
+    if (cli)
+        return cli;
+    const suite = suiteJudgeProfile?.trim();
+    if (suite)
+        return suite;
+    return undefined;
+}
+/** Best-effort model name for the self-describing judge line. `BaseChatModel` doesn't standardize a
+ * public model-name field, so read `model` then `modelName`; return `undefined` if neither is a
+ * usable string (the profile name alone still describes the run). */
+function judgeModelName(config) {
+    const llm = config.llm;
+    const name = typeof llm.model === 'string' ? llm.model : llm.modelName;
+    return typeof name === 'string' && name.length > 0 ? name : undefined;
+}
+/**
+ * Adds the `eval` command to the program.
+ *
+ * `gth eval <suite.yaml> [-j 8] [-o out/]` — grades a suite of cases (deterministic checks and/or
+ * an LLM judge) against the SUT agent, the way `review` grades a diff. See
+ * docs/batch-eval-cli-surface.md and docs/batch-eval-user-requirements.md for the design; this
+ * command implements the single-`prompt`, `gth-agent`-target, `config.llm`-as-judge subset — see
+ * the BATCH-2 Task 2 brief's "Not in scope" list for what's deliberately deferred.
+ *
+ * Exit-code contract (BATCH-11 / #405 his #6 — a distinct three-way code so CI can tell a product
+ * regression from a broken harness):
+ * - `0` — every case passed (mirrors `review`; unchanged contract).
+ * - `1` — the suite **ran** but ≥1 case FAILED (a deterministic check or the judge fell below
+ *   threshold). A *product* signal: real, gradeable results, some below the bar.
+ * - `2` — **harness error**: the suite couldn't be meaningfully evaluated — the suite file failed
+ *   to load/parse, config failed to build, or **every** case's SUT run failed (`sutOk:false`, e.g.
+ *   a transport/auth/config failure produced no answer to grade). An *environment* signal.
+ *
+ * The suite-vs-harness split is anchored on `sutOk` (see `classifyEvalExit`), not the verdict: a
+ * case that ran (`sutOk:true`) but whose judge errored, or whose answer failed a check, is still a
+ * real result → exit `1`, never `2`. Unlike `batch`, which exits 0 regardless of per-cell quality.
+ *
+ * @param program - The commander program
+ * @param commandLineConfigOverrides - command line config overrides
+ */
+export function evalCommand(program, commandLineConfigOverrides) {
+    program
+        .command('eval')
+        .description('Grade a suite of cases against the SUT agent with deterministic checks and/or an LLM ' +
+        'judge ("pytest for prompts"). Exit code: 0 = all cases passed, 1 = a case failed, ' +
+        '2 = harness error (bad suite/config, or the SUT could not run).')
+        .argument('<suites...>', 'Eval suite YAML file(s) or directory(ies). A directory runs its direct-child *.yaml/*.yml ' +
+        'suites (non-recursive, sorted). Multiple suites report ONE aggregate exit code; each ' +
+        'writes into its own <output>/<suite-name>/ subdir (a single suite writes into <output> directly).')
+        .option('-j, --concurrency <n>', 'Max in-flight cases', parseIntOption)
+        .option('-o, --output <dir>', 'Directory to write structured per-case JSON + results.json summary to ' +
+        '(default: a timestamped dir alongside other gth reports)')
+        .option('--judge <profile>', 'Identity profile whose model judges the cases (defaults to the SUT model). ' +
+        'Overrides the suite-level judge_profile.')
+        .option('-r, --reporter <names>', 'Reporter(s) to render the run through (repeatable; comma-separated). Built-in: "text" ' +
+        '(default), "junit" (writes results.xml). REPLACES the default, so "--reporter junit" is ' +
+        'JUnit only. The always-on results.json + per-cell JSON are written regardless.', (value, previous = []) => [...previous, value])
+        .addHelpText('after', '\n' +
+        'Reporters:\n' +
+        '  text (default)  a human-readable PASS/FAIL summary on the console\n' +
+        '  junit           an Ant-JUnit results.xml (for TeamCity / CI JUnit readers)\n' +
+        '  Custom reporters are declared in config under `reporters: { <name>: <module path> }`.\n' +
+        '  --reporter REPLACES the default set (it does not add to it); pass every reporter you\n' +
+        '  want, e.g. `--reporter text,junit`. results.json is always written, regardless.\n' +
+        '\n' +
+        'Examples:\n' +
+        '  $ gsloth eval eval/js-basics.yaml\n' +
+        '  $ gsloth eval eval/js-basics.yaml --judge strict-judge\n' +
+        '  $ gsloth eval eval/authz-matrix.yaml -j 8 -o eval/out/authz\n' +
+        '  $ gsloth eval eval/js-basics.yaml eval/authz-matrix.yaml   # many suites, one exit\n' +
+        '  $ gsloth eval eval/ -o eval/out --reporter junit           # a whole directory\n' +
+        '  $ gsloth eval eval/js-basics.yaml --reporter junit\n' +
+        '  $ gsloth eval eval/js-basics.yaml --reporter text,junit\n')
+        .action(async (suitePaths, options) => {
+        try {
+            // Specific `.js` subpaths (not the bare package root), matching batchCommand.ts's
+            // convention — vitest's workspace-import resolver recognizes these and resolves straight to
+            // source, so specs exercise live `packages/batch/src` rather than a `dist/` build.
+            const { parseEvalSuite } = await import('@gaunt-sloth/batch/evalSuite.js');
+            const { runEvalSuite, classifyEvalExit } = await import('@gaunt-sloth/batch/evalRunner.js');
+            const { writeEvalOutput } = await import('@gaunt-sloth/batch/evalOutput.js');
+            const { resolveReporters } = await import('@gaunt-sloth/batch/reporters/registry.js');
+            const { driveReporters } = await import('@gaunt-sloth/batch/reporters/drive.js');
+            // The reporter selection (`--reporter`, else the default `['text']`) and the output ROOT are
+            // invocation-level — the same for every suite. REPLACES the default: the `--reporter` value
+            // when given, else `['text']`.
+            const reporterNames = options.reporter
+                ? normalizeReporterNames(options.reporter)
+                : ['text'];
+            const outputRoot = options.output ?? defaultEvalOutputDir();
+            // BATCH-19 Task B: run ONE whole suite — parse, precondition-check its identities, build its
+            // configs/reporters, run, write its structured output, and drive its reporters — RETURNING
+            // its summary. THROWS on this suite's own harness error (parse/config/reporter/precondition
+            // failure), which the caller catches per suite so one bad suite never aborts the good ones.
+            //
+            // Task B, Deliverable 2 — the `-i` papercut: a MATRIX suite (declares `identities:`) must run
+            // with NO base `-i`. The per-identity configs come from the suite's `identities:` list (each
+            // pre-validated by the precondition below), so the base config used for reporters + the
+            // default judge is the FIRST identity's already-built config — the matrix path NEVER calls
+            // `initConfig` without an `identityProfile`. That is the fix: the old top-level
+            // `initConfig(overrides)` demanded a resolvable base config (a matrix-only project has none),
+            // and its terminal `exit(1)` is uncatchable, so it can't be reached at all on this path. The
+            // NON-matrix path is unchanged — the base config is still `initConfig(overrides)`. (Configs
+            // are built per suite, not cached across suites — a documented non-goal.)
+            const runOneSuite = async (suite, suiteOutputDir) => {
+                const suiteText = readFileFromProjectDir(suite.readPath);
+                const parsedSuite = parseEvalSuite(suiteText, suite.readPath);
+                // BATCH-12 PRECONDITION — trustworthy loads (no false-green): every suite-declared identity
+                // must resolve to its OWN config (`.gsloth-settings/<name>/`), not the global/plain
+                // fallback. Verify ALL of them with GS2-62's PURE, catchable helper BEFORE building any
+                // per-identity config, because handing a bad profile to `initConfig({ …, identityProfile })`
+                // would hit its uncatchable `exit(1)` and collapse the harness-vs-product (2-vs-1) exit-code
+                // distinction. A silent wrong-identity run is the worst failure for an auth matrix — so an
+                // unresolved identity throws here → per-suite catch → exit 2, and NOTHING runs.
+                const suiteIdentities = parsedSuite.identities ?? [];
+                const unresolvedIdentities = suiteIdentities.filter((identity) => !resolveIdentityProfileConfigPath(identity));
+                if (unresolvedIdentities.length > 0) {
+                    throw new Error(`identity profile(s) not found: ${unresolvedIdentities.join(', ')} — each suite ` +
+                        '`identities` entry must have its own config file in ' +
+                        '.gsloth/.gsloth-settings/<name>/. No cases were run.');
+                }
+                const runCellOptions = {
+                    command: 'ask',
+                    sourcePrefix: 'EVAL',
+                    wrapBlockPrefix: 'message',
+                    wrapPrefix: 'user message',
+                };
+                // Build the SUT run function(s), selected by `target.type`, AND resolve the base config
+                // used for the reporters + the default (SUT-model) judge. The eval runner is
+                // target-agnostic — it just consumes an injected single-turn `runCell` and multi-turn
+                // `runConversation` — so the target only changes which builders produce them.
+                let runCell;
+                let runConversation;
+                let runCellByIdentity;
+                let runConversationByIdentity;
+                let baseConfig;
+                if (suiteIdentities.length > 0 && parsedSuite.target.type === 'gth-agent') {
+                    // MATRIX (gth-agent): one runCell per identity, each from a fresh
+                    // `initConfig({ …, identityProfile })` (mirrors `gth batch --models`' per-model
+                    // construction — a genuinely fresh `.llm`, never a structural clone), built ONCE per
+                    // identity and reused across cases by the runner. An identity profile's manual
+                    // `mcpServers.<n>.headers.Authorization` (CFG-4) flows through as-is. The base config for
+                    // reporters + the default judge is the FIRST identity's config (reused, not rebuilt) — so
+                    // this path makes NO `initConfig` call without an `identityProfile` (Deliverable 2).
+                    runCellByIdentity = new Map();
+                    runConversationByIdentity = new Map();
+                    let firstIdentityConfig;
+                    for (const identity of suiteIdentities) {
+                        const identityConfig = await initConfig({
+                            ...commandLineConfigOverrides,
+                            identityProfile: identity,
+                        });
+                        if (!firstIdentityConfig)
+                            firstIdentityConfig = identityConfig;
+                        const identityPreamble = getAskSystemPrompt(identityConfig);
+                        runCellByIdentity.set(identity, await buildProductionRunCell(identityConfig, identityPreamble, commandLineConfigOverrides, runCellOptions));
+                        runConversationByIdentity.set(identity, await buildProductionRunConversation(identityConfig, identityPreamble, runCellOptions));
+                    }
+                    // `firstIdentityConfig` is always set here (suiteIdentities is non-empty).
+                    baseConfig = firstIdentityConfig;
+                }
+                else if (parsedSuite.target.type === 'adk-agent') {
+                    // BATCH-14: drive an EXTERNAL Google ADK agent over A2A. The agent runs out-of-process
+                    // (its own model/tools/auth), so there is no per-identity gth config — the `identities`
+                    // matrix is rejected for this target at parse time, leaving only the single-run path. The
+                    // judge still grades via the local gth config, so build it from `initConfig(overrides)`.
+                    baseConfig = await initConfig(commandLineConfigOverrides);
+                    const { buildAdkRunCell, buildAdkRunConversation } = await import('#src/commands/adkEvalRunner.js');
+                    runCell = buildAdkRunCell(parsedSuite.target);
+                    runConversation = buildAdkRunConversation(parsedSuite.target);
+                }
+                else if (parsedSuite.target.type === 'ag-ui') {
+                    // BATCH-15: drive an EXTERNAL AG-UI agent over HTTP/SSE. Like the ADK target, the agent
+                    // runs out-of-process (its own model/tools/auth), so there is no per-identity gth config —
+                    // the `identities` matrix is rejected for this target at parse time. Unlike ADK, the AG-UI
+                    // wire streams tool calls, so the runner captures them and `must_call`/`must_not_call`
+                    // grade normally. The judge still grades via the local gth `initConfig(overrides)` config.
+                    baseConfig = await initConfig(commandLineConfigOverrides);
+                    const { buildAgUiRunCell, buildAgUiRunConversation } = await import('#src/commands/agUiEvalRunner.js');
+                    runCell = buildAgUiRunCell(parsedSuite.target);
+                    runConversation = buildAgUiRunConversation(parsedSuite.target);
+                }
+                else {
+                    // gth-agent, NO identities (unchanged): one runCell/runConversation under the invoked
+                    // profile, from the single `initConfig(overrides)` base config (byte-for-byte as before).
+                    baseConfig = await initConfig(commandLineConfigOverrides);
+                    const preamble = getAskSystemPrompt(baseConfig);
+                    runCell = await buildProductionRunCell(baseConfig, preamble, commandLineConfigOverrides, runCellOptions);
+                    runConversation = await buildProductionRunConversation(baseConfig, preamble, runCellOptions);
+                }
+                // BATCH-19 A2: resolve the reporter selection — BEFORE the suite runs and before any output
+                // is written — so an unknown `--reporter` name (or a broken config-reporter module) fails
+                // fast → this suite's harness error (exit 2) with NOTHING run and no misleading partial
+                // output. The bundled JUnit reporter and any config-declared reporters are registered
+                // through the ONE public `custom` seam `resolveReporters` exposes (config wins on a name
+                // collision). Reporters are re-resolved per suite so each suite gets fresh reporter state
+                // (e.g. its own JUnit document). An unknown name throws (message lists the available
+                // reporters) → per-suite catch → exit 2.
+                const customReporterFactories = await buildCustomReporterFactories(baseConfig);
+                const reporters = resolveReporters(reporterNames, customReporterFactories);
+                // BATCH-10 Task 2: resolve the judge identity profile (CLI `--judge` > suite `judge_profile`
+                // > none). When set, build a SEPARATE config for it — the same supported
+                // `initConfig({ …overrides, identityProfile })` path `gth batch --models` uses to
+                // reconstruct a fresh `.llm` — so the judge runs under its own model. When unset, the judge
+                // shares the base config (the SUT's for a single run; the FIRST identity's for a matrix).
+                const judgeProfile = resolveJudgeProfile(options.judge, parsedSuite.judgeProfile);
+                if (judgeProfile && !resolveIdentityProfileConfigPath(judgeProfile)) {
+                    // GS2-62 (BATCH-10 review Minor 1): pre-check the requested judge profile with the PURE
+                    // helper and throw our OWN catchable error. Without this, `initConfig({ …, identityProfile
+                    // })` would silently fall back to the global (or plain) config on a bad profile and the
+                    // self-describing notice below would print `Judge: profile "<typo>" (model: <global>)` —
+                    // asserting a profile that never loaded. The throw is caught per suite → exit 2 (harness
+                    // error). We can't lean on initConfig's own hard-error here: that path calls the
+                    // uncatchable `exit(1)`, which would end the process with code 1 and collapse the
+                    // harness-vs-product (2-vs-1) exit-code distinction.
+                    throw new Error(`judge profile "${judgeProfile}" not found: no config file in ` +
+                        `.gsloth/.gsloth-settings/${judgeProfile}/`);
+                }
+                const judgeConfig = judgeProfile
+                    ? await initConfig({ ...commandLineConfigOverrides, identityProfile: judgeProfile })
+                    : baseConfig;
+                const judge = buildProductionJudge(judgeConfig);
+                const summary = await runEvalSuite(parsedSuite, {
+                    runCell,
+                    runCellByIdentity,
+                    runConversation,
+                    runConversationByIdentity,
+                    judge,
+                    concurrency: options.concurrency,
+                });
+                writeEvalOutput(suiteOutputDir, summary);
+                // BATCH-19: render the run through the reporters resolved above (A1 seam + A2 selection).
+                // `writeEvalOutput` (results.json + per-cell JSON) is always-on core output, independent of
+                // any reporter — a JUnit run writes results.xml IN ADDITION, never instead. Reporters are an
+                // additional rendering layer, driven here in lifecycle order, with THIS suite's dir + path.
+                await driveReporters(reporters, summary, {
+                    suitePath: suite.readPath,
+                    outputDir: suiteOutputDir,
+                    judgeNotice: judgeProfile
+                        ? { profile: judgeProfile, model: judgeModelName(judgeConfig) }
+                        : undefined,
+                });
+                return summary;
+            };
+            // Deliverable 1: resolve the CLI args into the ordered, de-duplicated suite list (files as
+            // given; directories → their sorted direct-child *.yaml/*.yml). A no-match directory records
+            // a harness error but does not abort the good inputs.
+            const { suites, errors: resolutionErrors } = resolveEvalSuiteInputs(suitePaths);
+            let anyHarnessError = false;
+            for (const message of resolutionErrors) {
+                displayError(`eval harness error: ${message}`);
+                anyHarnessError = true;
+            }
+            // Exactly one resolved suite → write DIRECTLY into the output dir (byte-for-byte with the
+            // single-suite contract). Multiple → each suite writes into `<output>/<suiteStem>/`, so a CI
+            // glob collects them and suites never clobber. On a stem collision across resolved suites,
+            // disambiguate deterministically (`-2`, `-3`, … in resolved order) and WARN — never silently
+            // overwrite.
+            const single = suites.length === 1;
+            const usedStems = new Map();
+            const combinedCases = [];
+            for (const suite of suites) {
+                let suiteOutputDir = outputRoot;
+                if (!single) {
+                    const stem = suiteStem(suite.readPath);
+                    const priorCount = usedStems.get(stem) ?? 0;
+                    usedStems.set(stem, priorCount + 1);
+                    const dirName = priorCount === 0 ? stem : `${stem}-${priorCount + 1}`;
+                    if (priorCount > 0) {
+                        displayWarning(`eval: two suites share the output name "${stem}"; writing "${suite.readPath}" ` +
+                            `output to "${dirName}/" so it does not clobber the earlier one.`);
+                    }
+                    suiteOutputDir = join(outputRoot, dirName);
+                }
+                try {
+                    const summary = await runOneSuite(suite, suiteOutputDir);
+                    combinedCases.push(...summary.cases);
+                }
+                catch (error) {
+                    // A harness-level error for THIS suite (parse/config/reporter/precondition/unexpected).
+                    // A harness error in ANY suite dominates the aggregate exit → 2 (a partial run can't be
+                    // trusted), but the good suites still ran and wrote their output.
+                    displayError(`eval harness error${single ? '' : ` in ${suite.readPath}`}: ` +
+                        (error instanceof Error ? error.message : String(error)));
+                    anyHarnessError = true;
+                }
+            }
+            // Aggregate three-way exit: concatenate every successful suite's cases into ONE combined
+            // summary and call `classifyEvalExit` ONCE. A harness error in ANY suite dominates → exit 2.
+            // For N=1 this reproduces the single-suite exit EXACTLY (one summary, or one harness error →
+            // 2); `classifyEvalExit`'s own empty/all-`!sutOk` rule also yields 2 when nothing gradeable
+            // ran anywhere.
+            const passed = combinedCases.filter((result) => result.verdict === 'PASS').length;
+            const combined = {
+                total: combinedCases.length,
+                passed,
+                failed: combinedCases.length - passed,
+                cases: combinedCases,
+            };
+            // Multi-suite only: one aggregate total line AFTER every suite's own per-suite output. For
+            // N=1 print NOTHING extra — the single suite's text reporter already printed its verdict, so
+            // the output stays byte-for-byte identical to the single-suite contract.
+            if (suites.length > 1) {
+                const totalLine = `EVAL TOTAL: ${combined.passed}/${combined.total} across ${suites.length} suites, ` +
+                    `${combined.failed} failed`;
+                if (combined.failed === 0 && !anyHarnessError) {
+                    displaySuccess(totalLine);
+                }
+                else {
+                    displayWarning(totalLine);
+                }
+            }
+            const exitCode = anyHarnessError ? 2 : classifyEvalExit(combined);
+            if (exitCode !== 0) {
+                setExitCode(exitCode);
+            }
+        }
+        catch (error) {
+            // BATCH-11: an invocation-level harness error the run never recovered from (e.g. a failed
+            // dynamic import) — distinct from a per-suite error (caught in the loop). The suites couldn't
+            // be meaningfully evaluated, so this is an *environment* signal (exit 2). Surface the reason;
+            // do NOT rethrow — an uncaught throw would surface as a generic exit 1 via the entry point,
+            // collapsing the 1-vs-2 distinction.
+            displayError(`eval harness error: ${error instanceof Error ? error.message : String(error)}`);
+            setExitCode(2);
+        }
+    });
+}
+//# sourceMappingURL=evalCommand.js.map

package/dist/commands/evalCommand.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evalCommand.js","sourceRoot":"","sources":["../../src/commands/evalCommand.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAEL,UAAU,EACV,gCAAgC,GACjC,MAAM,6BAA6B,CAAC;AAErC,OAAO,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC;AAC3E,OAAO,EACL,sBAAsB,EACtB,8BAA8B,GAC/B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AACnE,OAAO,EACL,YAAY,EACZ,cAAc,EACd,cAAc,GACf,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,wCAAwC,CAAC;AACpF,OAAO,EACL,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,GACvB,MAAM,sCAAsC,CAAC;AAwB9C;kGACkG;AAClG,MAAM,UAAU,sBAAsB,CAAC,SAAmB;IACxD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC5B,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YACvD,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACf,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,4BAA4B,CACzC,MAAiB;IAEjB,MAAM,MAAM,GAAwC,EAAE,CAAC;IAEvD,MAAM,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,GAChD,MAAM,MAAM,CAAC,2CAA2C,CAAC,CAAC;IAC5D,MAAM,CAAC,mBAAmB,CAAC,GAAG,mBAAmB,CAAC;IAElD,KAAK,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,EAAE,CAAC;QACxE,MAAM,OAAO,GAAG,OAAO,CAAC,aAAa,EAAE,EAAE,UAAU,CAAC,CAAC;QACrD,IAAI,GAA4B,CAAC;QACjC,IAAI,CAAC;YACH,GAAG,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAC1C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,mCAAmC,IAAI,WAAW,UAAU,MAAM,OAAO,KAAK;gBAC5E,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAC3D,CAAC;QACJ,CAAC;QACD,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;QAC5B,IAAI,OAAO,OAAO,KAAK,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CACb,oBAAoB,IAAI,MAAM,UAAU,mCAAmC;gBACzE,qDAAqD,OAAO,OAAO,GAAG,CACzE,CAAC;QACJ,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,GAAG,OAA8B,CAAC;IAChD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;8FAC8F;AAC9F,MAAM,UAAU,oBAAoB;IAClC,OAAO,iBAAiB,CAAC,OAAO,iBAAiB,EAAE,OAAO,CAAC,CAAC;AAC9D,CAAC;AAWD;qEACqE;AACrE,SAAS,SAAS,CAAC,IAAY;IAC7B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5B,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1B,OAAO,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACjD,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAS,sBAAsB,CAAC,MAAgB;IAI9C,MAAM,UAAU,GAAG,aAAa,EAAE,CAAC;IACnC,MAAM,MAAM,GAAwB,EAAE,CAAC;IACvC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,MAAM,IAAI,GAAG,CAAC,QAAgB,EAAE,OAAe,EAAQ,EAAE;QACvD,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;YAAE,OAAO;QAC9B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClB,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;IACrC,CAAC,CAAC;IAEF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QACvC,IAAI,WAAW,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC;YACH,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;QAC5C,CAAC;QAAC,MAAM,CAAC;YACP,6FAA6F;YAC7F,6FAA6F;YAC7F,WAAW,GAAG,KAAK,CAAC;QACtB,CAAC;QAED,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YACjB,SAAS;QACX,CAAC;QAED,MAAM,YAAY,GAAG,WAAW,CAAC,GAAG,CAAC;aAClC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;YACf,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;YACxC,OAAO,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,MAAM,CAAC;QAC3C,CAAC,CAAC;aACD,IAAI,EAAE,CAAC;QACV,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,MAAM,CAAC,IAAI,CAAC,yBAAyB,KAAK,gCAAgC,CAAC,CAAC;YAC5E,SAAS;QACX,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;YAChC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YACpC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;AAC5B,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,oBAAoB,CAAC,MAAiB;IAC7C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;QAC9B,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;QACtE,OAAO,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IACnD,CAAC,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CACjC,QAA4B,EAC5B,iBAAqC;IAErC,MAAM,GAAG,GAAG,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC7B,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IACpB,MAAM,KAAK,GAAG,iBAAiB,EAAE,IAAI,EAAE,CAAC;IACxC,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC;IACxB,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;qEAEqE;AACrE,SAAS,cAAc,CAAC,MAAiB;IACvC,MAAM,GAAG,GAAG,MAAM,CAAC,GAA+C,CAAC;IACnE,MAAM,IAAI,GAAG,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC;IACvE,OAAO,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AACxE,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,UAAU,WAAW,CACzB,OAAgB,EAChB,0BAAsD;IAEtD,OAAO;SACJ,OAAO,CAAC,MAAM,CAAC;SACf,WAAW,CACV,uFAAuF;QACrF,oFAAoF;QACpF,iEAAiE,CACpE;SACA,QAAQ,CACP,aAAa,EACb,4FAA4F;QAC1F,uFAAuF;QACvF,mGAAmG,CACtG;SACA,MAAM,CAAC,uBAAuB,EAAE,qBAAqB,EAAE,cAAc,CAAC;SACtE,MAAM,CACL,oBAAoB,EACpB,wEAAwE;QACtE,0DAA0D,CAC7D;SACA,MAAM,CACL,mBAAmB,EACnB,6EAA6E;QAC3E,0CAA0C,CAC7C;SACA,MAAM,CACL,wBAAwB,EACxB,wFAAwF;QACtF,0FAA0F;QAC1F,gFAAgF,EAClF,CAAC,KAAa,EAAE,QAAQ,GAAa,EAAE,EAAE,EAAE,CAAC,CAAC,GAAG,QAAQ,EAAE,KAAK,CAAC,CACjE;SACA,WAAW,CACV,OAAO,EACP,IAAI;QACF,cAAc;QACd,uEAAuE;QACvE,gFAAgF;QAChF,2FAA2F;QAC3F,0FAA0F;QAC1F,qFAAqF;QACrF,IAAI;QACJ,aAAa;QACb,uCAAuC;QACvC,4DAA4D;QAC5D,iEAAiE;QACjE,wFAAwF;QACxF,oFAAoF;QACpF,wDAAwD;QACxD,6DAA6D,CAChE;SACA,MAAM,CAAC,KAAK,EAAE,UAAoB,EAAE,OAA2B,EAAE,EAAE;QAClE,IAAI,CAAC;YACH,kFAAkF;YAClF,4FAA4F;YAC5F,mFAAmF;YACnF,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,iCAAiC,CAAC,CAAC;YAC3E,MAAM,EAAE,YAAY,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,kCAAkC,CAAC,CAAC;YAC5F,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,kCAAkC,CAAC,CAAC;YAC7E,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,0CAA0C,CAAC,CAAC;YACtF,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,uCAAuC,CAAC,CAAC;YAEjF,6FAA6F;YAC7F,4FAA4F;YAC5F,+BAA+B;YAC/B,MAAM,aAAa,GAAG,OAAO,CAAC,QAAQ;gBACpC,CAAC,CAAC,sBAAsB,CAAC,OAAO,CAAC,QAAQ,CAAC;gBAC1C,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACb,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,IAAI,oBAAoB,EAAE,CAAC;YAE5D,6FAA6F;YAC7F,2FAA2F;YAC3F,4FAA4F;YAC5F,4FAA4F;YAC5F,EAAE;YACF,8FAA8F;YAC9F,6FAA6F;YAC7F,wFAAwF;YACxF,2FAA2F;YAC3F,gFAAgF;YAChF,8FAA8F;YAC9F,6FAA6F;YAC7F,4FAA4F;YAC5F,0EAA0E;YAC1E,MAAM,WAAW,GAAG,KAAK,EACvB,KAAwB,EACxB,cAAsB,EACK,EAAE;gBAC7B,MAAM,SAAS,GAAG,sBAAsB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACzD,MAAM,WAAW,GAAG,cAAc,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;gBAE9D,4FAA4F;gBAC5F,oFAAoF;gBACpF,wFAAwF;gBACxF,6FAA6F;gBAC7F,6FAA6F;gBAC7F,2FAA2F;gBAC3F,gFAAgF;gBAChF,MAAM,eAAe,GAAG,WAAW,CAAC,UAAU,IAAI,EAAE,CAAC;gBACrD,MAAM,oBAAoB,GAAG,eAAe,CAAC,MAAM,CACjD,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,gCAAgC,CAAC,QAAQ,CAAC,CAC1D,CAAC;gBACF,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACpC,MAAM,IAAI,KAAK,CACb,kCAAkC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB;wBAC/E,sDAAsD;wBACtD,sDAAsD,CACzD,CAAC;gBACJ,CAAC;gBAED,MAAM,cAAc,GAAG;oBACrB,OAAO,EAAE,KAAK;oBACd,YAAY,EAAE,MAAM;oBACpB,eAAe,EAAE,SAAS;oBAC1B,UAAU,EAAE,cAAc;iBAClB,CAAC;gBAEX,wFAAwF;gBACxF,6EAA6E;gBAC7E,sFAAsF;gBACtF,8EAA8E;gBAC9E,IAAI,OAA8B,CAAC;gBACnC,IAAI,eAA8C,CAAC;gBACnD,IAAI,iBAAqD,CAAC;gBAC1D,IAAI,yBAAqE,CAAC;gBAC1E,IAAI,UAAqB,CAAC;gBAC1B,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;oBAC1E,kEAAkE;oBAClE,gFAAgF;oBAChF,qFAAqF;oBACrF,+EAA+E;oBAC/E,0FAA0F;oBAC1F,0FAA0F;oBAC1F,qFAAqF;oBACrF,iBAAiB,GAAG,IAAI,GAAG,EAAE,CAAC;oBAC9B,yBAAyB,GAAG,IAAI,GAAG,EAAE,CAAC;oBACtC,IAAI,mBAA0C,CAAC;oBAC/C,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE,CAAC;wBACvC,MAAM,cAAc,GAAG,MAAM,UAAU,CAAC;4BACtC,GAAG,0BAA0B;4BAC7B,eAAe,EAAE,QAAQ;yBAC1B,CAAC,CAAC;wBACH,IAAI,CAAC,mBAAmB;4BAAE,mBAAmB,GAAG,cAAc,CAAC;wBAC/D,MAAM,gBAAgB,GAAG,kBAAkB,CAAC,cAAc,CAAC,CAAC;wBAC5D,iBAAiB,CAAC,GAAG,CACnB,QAAQ,EACR,MAAM,sBAAsB,CAC1B,cAAc,EACd,gBAAgB,EAChB,0BAA0B,EAC1B,cAAc,CACf,CACF,CAAC;wBACF,yBAAyB,CAAC,GAAG,CAC3B,QAAQ,EACR,MAAM,8BAA8B,CAClC,cAAc,EACd,gBAAgB,EAChB,cAAc,CACf,CACF,CAAC;oBACJ,CAAC;oBACD,2EAA2E;oBAC3E,UAAU,GAAG,mBAAoB,CAAC;gBACpC,CAAC;qBAAM,IAAI,WAAW,CAAC,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;oBACnD,uFAAuF;oBACvF,wFAAwF;oBACxF,0FAA0F;oBAC1F,yFAAyF;oBACzF,UAAU,GAAG,MAAM,UAAU,CAAC,0BAA0B,CAAC,CAAC;oBAC1D,MAAM,EAAE,eAAe,EAAE,uBAAuB,EAAE,GAChD,MAAM,MAAM,CAAC,gCAAgC,CAAC,CAAC;oBACjD,OAAO,GAAG,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;oBAC9C,eAAe,GAAG,uBAAuB,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBAChE,CAAC;qBAAM,IAAI,WAAW,CAAC,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBAC/C,wFAAwF;oBACxF,2FAA2F;oBAC3F,2FAA2F;oBAC3F,uFAAuF;oBACvF,2FAA2F;oBAC3F,UAAU,GAAG,MAAM,UAAU,CAAC,0BAA0B,CAAC,CAAC;oBAC1D,MAAM,EAAE,gBAAgB,EAAE,wBAAwB,EAAE,GAClD,MAAM,MAAM,CAAC,iCAAiC,CAAC,CAAC;oBAClD,OAAO,GAAG,gBAAgB,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;oBAC/C,eAAe,GAAG,wBAAwB,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBACjE,CAAC;qBAAM,CAAC;oBACN,sFAAsF;oBACtF,0FAA0F;oBAC1F,UAAU,GAAG,MAAM,UAAU,CAAC,0BAA0B,CAAC,CAAC;oBAC1D,MAAM,QAAQ,GAAG,kBAAkB,CAAC,UAAU,CAAC,CAAC;oBAChD,OAAO,GAAG,MAAM,sBAAsB,CACpC,UAAU,EACV,QAAQ,EACR,0BAA0B,EAC1B,cAAc,CACf,CAAC;oBACF,eAAe,GAAG,MAAM,8BAA8B,CACpD,UAAU,EACV,QAAQ,EACR,cAAc,CACf,CAAC;gBACJ,CAAC;gBAED,4FAA4F;gBAC5F,0FAA0F;gBAC1F,wFAAwF;gBACxF,sFAAsF;gBACtF,yFAAyF;gBACzF,0FAA0F;gBAC1F,qFAAqF;gBACrF,yCAAyC;gBACzC,MAAM,uBAAuB,GAAG,MAAM,4BAA4B,CAAC,UAAU,CAAC,CAAC;gBAC/E,MAAM,SAAS,GAAG,gBAAgB,CAAC,aAAa,EAAE,uBAAuB,CAAC,CAAC;gBAE3E,6FAA6F;gBAC7F,yEAAyE;gBACzE,kFAAkF;gBAClF,4FAA4F;gBAC5F,0FAA0F;gBAC1F,MAAM,YAAY,GAAG,mBAAmB,CAAC,OAAO,CAAC,KAAK,EAAE,WAAW,CAAC,YAAY,CAAC,CAAC;gBAClF,IAAI,YAAY,IAAI,CAAC,gCAAgC,CAAC,YAAY,CAAC,EAAE,CAAC;oBACpE,wFAAwF;oBACxF,2FAA2F;oBAC3F,wFAAwF;oBACxF,yFAAyF;oBACzF,yFAAyF;oBACzF,iFAAiF;oBACjF,kFAAkF;oBAClF,qDAAqD;oBACrD,MAAM,IAAI,KAAK,CACb,kBAAkB,YAAY,iCAAiC;wBAC7D,4BAA4B,YAAY,GAAG,CAC9C,CAAC;gBACJ,CAAC;gBACD,MAAM,WAAW,GAAG,YAAY;oBAC9B,CAAC,CAAC,MAAM,UAAU,CAAC,EAAE,GAAG,0BAA0B,EAAE,eAAe,EAAE,YAAY,EAAE,CAAC;oBACpF,CAAC,CAAC,UAAU,CAAC;gBACf,MAAM,KAAK,GAAG,oBAAoB,CAAC,WAAW,CAAC,CAAC;gBAEhD,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,WAAW,EAAE;oBAC9C,OAAO;oBACP,iBAAiB;oBACjB,eAAe;oBACf,yBAAyB;oBACzB,KAAK;oBACL,WAAW,EAAE,OAAO,CAAC,WAAW;iBACjC,CAAC,CAAC;gBAEH,eAAe,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;gBAEzC,0FAA0F;gBAC1F,4FAA4F;gBAC5F,6FAA6F;gBAC7F,4FAA4F;gBAC5F,MAAM,cAAc,CAAC,SAAS,EAAE,OAAO,EAAE;oBACvC,SAAS,EAAE,KAAK,CAAC,QAAQ;oBACzB,SAAS,EAAE,cAAc;oBACzB,WAAW,EAAE,YAAY;wBACvB,CAAC,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,CAAC,WAAW,CAAC,EAAE;wBAC/D,CAAC,CAAC,SAAS;iBACd,CAAC,CAAC;gBAEH,OAAO,OAAO,CAAC;YACjB,CAAC,CAAC;YAEF,2FAA2F;YAC3F,6FAA6F;YAC7F,sDAAsD;YACtD,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,EAAE,GAAG,sBAAsB,CAAC,UAAU,CAAC,CAAC;YAChF,IAAI,eAAe,GAAG,KAAK,CAAC;YAC5B,KAAK,MAAM,OAAO,IAAI,gBAAgB,EAAE,CAAC;gBACvC,YAAY,CAAC,uBAAuB,OAAO,EAAE,CAAC,CAAC;gBAC/C,eAAe,GAAG,IAAI,CAAC;YACzB,CAAC;YAED,0FAA0F;YAC1F,6FAA6F;YAC7F,2FAA2F;YAC3F,6FAA6F;YAC7F,aAAa;YACb,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC;YACnC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;YAC5C,MAAM,aAAa,GAAqB,EAAE,CAAC;YAC3C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,IAAI,cAAc,GAAG,UAAU,CAAC;gBAChC,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;oBACvC,MAAM,UAAU,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC5C,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC;oBACpC,MAAM,OAAO,GAAG,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;oBACtE,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;wBACnB,cAAc,CACZ,2CAA2C,IAAI,eAAe,KAAK,CAAC,QAAQ,IAAI;4BAC9E,cAAc,OAAO,4CAA4C,CACpE,CAAC;oBACJ,CAAC;oBACD,cAAc,GAAG,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;gBAC7C,CAAC;gBAED,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;oBACzD,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;gBACvC,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,wFAAwF;oBACxF,wFAAwF;oBACxF,kEAAkE;oBAClE,YAAY,CACV,qBAAqB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,QAAQ,EAAE,IAAI;wBAC5D,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAC3D,CAAC;oBACF,eAAe,GAAG,IAAI,CAAC;gBACzB,CAAC;YACH,CAAC;YAED,yFAAyF;YACzF,6FAA6F;YAC7F,6FAA6F;YAC7F,4FAA4F;YAC5F,gBAAgB;YAChB,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;YAClF,MAAM,QAAQ,GAAqB;gBACjC,KAAK,EAAE,aAAa,CAAC,MAAM;gBAC3B,MAAM;gBACN,MAAM,EAAE,aAAa,CAAC,MAAM,GAAG,MAAM;gBACrC,KAAK,EAAE,aAAa;aACrB,CAAC;YAEF,2FAA2F;YAC3F,6FAA6F;YAC7F,yEAAyE;YACzE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,SAAS,GACb,eAAe,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,KAAK,WAAW,MAAM,CAAC,MAAM,WAAW;oBACnF,GAAG,QAAQ,CAAC,MAAM,SAAS,CAAC;gBAC9B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;oBAC9C,cAAc,CAAC,SAAS,CAAC,CAAC;gBAC5B,CAAC;qBAAM,CAAC;oBACN,cAAc,CAAC,SAAS,CAAC,CAAC;gBAC5B,CAAC;YACH,CAAC;YAED,MAAM,QAAQ,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YAClE,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;gBACnB,WAAW,CAAC,QAAQ,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,0FAA0F;YAC1F,8FAA8F;YAC9F,8FAA8F;YAC9F,4FAA4F;YAC5F,qCAAqC;YACrC,YAAY,CACV,uBAAuB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAChF,CAAC;YACF,WAAW,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/commands/execCommand.js

@@ -94,9 +94,12 @@ export function execCommand(program, commandLineConfigOverrides) {
         const execConfig = buildExecConfig(config, options);
         const { runSingleShot } = await import('@gaunt-sloth/core/runtime/singleShot.js');
         const { createResolvers } = await import('@gaunt-sloth/agent/resolvers.js');
+        const { resolveAgentFactory } = await import('@gaunt-sloth/agent/core/resolveAgentFactory.js');
         let ok = false;
         try {
-            ok = await runSingleShot('EXEC', getExecSystemPrompt(execConfig), content.join('\n'), execConfig, createResolvers(), 'exec');
+            ({ ok } = await runSingleShot('EXEC', getExecSystemPrompt(execConfig), content.join('\n'), execConfig, createResolvers(), 'exec', 
+            // exec defaults to the lean backend; an explicit config.agent.backend overrides it.
+            resolveAgentFactory(execConfig, 'lean')));
         }
         catch (error) {
             displayError(error instanceof Error ? error.message : String(error));

package/dist/commands/execCommand.js.map

@@ -1 +1 @@
-{"version":3,"file":"execCommand.js","sourceRoot":"","sources":["../../src/commands/execCommand.ts"],"names":[],"mappings":"AACA,OAAO,EAA8B,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAErF,OAAO,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAC5E,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,wCAAwC,CAAC;AACzF,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,yCAAyC,CAAC;AACvF,OAAO,EAAE,WAAW,EAAE,MAAM,qCAAqC,CAAC;AAClE,OAAO,EAAE,+BAA+B,EAAE,MAAM,sCAAsC,CAAC;AAqBvF;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,eAAe,CAAC,MAAiB,EAAE,OAA2B;IAC5E,MAAM,UAAU,GAAc;QAC5B,GAAG,MAAM;QACT,uEAAuE;QACvE,4BAA4B,EAAE,KAAK;QACnC,+EAA+E;QAC/E,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,KAAK;QACrD,6EAA6E;QAC7E,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC5F,CAAC;IAEF,+FAA+F;IAC/F,4FAA4F;IAC5F,6EAA6E;IAC7E,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,IAAI,UAAU,CAAC,GAAG,IAAI,aAAa,IAAI,UAAU,CAAC,GAAG,EAAE,CAAC;QAC1F,UAAU,CAAC,GAA0C,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAC3F,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,WAAW,CACzB,OAAgB,EAChB,0BAAsD;IAEtD,OAAO;SACJ,OAAO,CAAC,MAAM,CAAC;SACf,WAAW,CACV,8FAA8F,CAC/F;SACA,QAAQ,CAAC,UAAU,EAAE,mCAAmC,CAAC;SACzD,MAAM,CACL,sBAAsB,EACtB,kGAAkG,CACnG;SACA,MAAM,CACL,uBAAuB,EACvB,qEAAqE,CACtE;SACA,MAAM,CACL,4BAA4B,EAC5B,gEAAgE,EAChE,UAAU,CACX;SACA,MAAM,CACL,oBAAoB,EACpB,sFAAsF;QACpF,4DAA4D,EAC9D,CAAC,KAAa,EAAE,WAAqB,EAAE,EAAE,EAAE,CAAC,CAAC,GAAG,QAAQ,EAAE,KAAK,CAAC,CACjE;SACA,MAAM,CAAC,KAAK,EAAE,MAA0B,EAAE,OAA2B,EAAE,EAAE;QACxE,yFAAyF;QACzF,IAAI,MAAM,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;QACxF,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAE5D,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,gEAAgE;QAChE,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,WAAW,GAAG,+BAA+B,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAClE,IAAI,WAAW,EAAE,CAAC;gBAChB,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,0CAA0C;QAC1C,4DAA4D;QAC5D,qCAAqC;QACrC,oBAAoB;QACpB,MAAM,eAAe,GAAG,kBAAkB,EAAE,CAAC;QAC7C,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAClC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,0BAA0B,EAAE,IAAI,CAAC,CAAC,CAAC;QACzF,CAAC;aAAM,IAAI,MAAM,EAAE,CAAC;YAClB,MAAM,aAAa,GAAG,+BAA+B,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;YAChE,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,QAAQ,EAAE,0BAA0B,EAAE,IAAI,CAAC,CAAC,CAAC;QACvF,CAAC;aAAM,IAAI,eAAe,EAAE,CAAC;YAC3B,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,eAAe,EAAE,QAAQ,EAAE,0BAA0B,EAAE,IAAI,CAAC,CAAC,CAAC;QACzF,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACb,oGAAoG,CACrG,CAAC;QACJ,CAAC;QAED,IAAI,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpD,cAAc,CACZ,uDAAuD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;gBACpF,6DAA6D,CAChE,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAEpD,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,yCAAyC,CAAC,CAAC;QAClF,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,iCAAiC,CAAC,CAAC;QAE5E,IAAI,EAAE,GAAG,KAAK,CAAC;QACf,IAAI,CAAC;YACH,EAAE,GAAG,MAAM,aAAa,CACtB,MAAM,EACN,mBAAmB,CAAC,UAAU,CAAC,EAC/B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAClB,UAAU,EACV,eAAe,EAAE,EACjB,MAAM,CACP,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACrE,EAAE,GAAG,KAAK,CAAC;QACb,CAAC;QAED,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,WAAW,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}
+{"version":3,"file":"execCommand.js","sourceRoot":"","sources":["../../src/commands/execCommand.ts"],"names":[],"mappings":"AACA,OAAO,EAA8B,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAErF,OAAO,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AAC5E,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,wCAAwC,CAAC;AACzF,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,yCAAyC,CAAC;AACvF,OAAO,EAAE,WAAW,EAAE,MAAM,qCAAqC,CAAC;AAClE,OAAO,EAAE,+BAA+B,EAAE,MAAM,sCAAsC,CAAC;AAqBvF;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,eAAe,CAAC,MAAiB,EAAE,OAA2B;IAC5E,MAAM,UAAU,GAAc;QAC5B,GAAG,MAAM;QACT,uEAAuE;QACvE,4BAA4B,EAAE,KAAK;QACnC,+EAA+E;QAC/E,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,KAAK;QACrD,6EAA6E;QAC7E,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC5F,CAAC;IAEF,+FAA+F;IAC/F,4FAA4F;IAC5F,6EAA6E;IAC7E,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,IAAI,UAAU,CAAC,GAAG,IAAI,aAAa,IAAI,UAAU,CAAC,GAAG,EAAE,CAAC;QAC1F,UAAU,CAAC,GAA0C,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAC3F,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,WAAW,CACzB,OAAgB,EAChB,0BAAsD;IAEtD,OAAO;SACJ,OAAO,CAAC,MAAM,CAAC;SACf,WAAW,CACV,8FAA8F,CAC/F;SACA,QAAQ,CAAC,UAAU,EAAE,mCAAmC,CAAC;SACzD,MAAM,CACL,sBAAsB,EACtB,kGAAkG,CACnG;SACA,MAAM,CACL,uBAAuB,EACvB,qEAAqE,CACtE;SACA,MAAM,CACL,4BAA4B,EAC5B,gEAAgE,EAChE,UAAU,CACX;SACA,MAAM,CACL,oBAAoB,EACpB,sFAAsF;QACpF,4DAA4D,EAC9D,CAAC,KAAa,EAAE,QAAQ,GAAa,EAAE,EAAE,EAAE,CAAC,CAAC,GAAG,QAAQ,EAAE,KAAK,CAAC,CACjE;SACA,MAAM,CAAC,KAAK,EAAE,MAA0B,EAAE,OAA2B,EAAE,EAAE;QACxE,yFAAyF;QACzF,IAAI,MAAM,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;QACxF,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAE5D,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,gEAAgE;QAChE,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,WAAW,GAAG,+BAA+B,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAClE,IAAI,WAAW,EAAE,CAAC;gBAChB,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,0CAA0C;QAC1C,4DAA4D;QAC5D,qCAAqC;QACrC,oBAAoB;QACpB,MAAM,eAAe,GAAG,kBAAkB,EAAE,CAAC;QAC7C,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAClC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,0BAA0B,EAAE,IAAI,CAAC,CAAC,CAAC;QACzF,CAAC;aAAM,IAAI,MAAM,EAAE,CAAC;YAClB,MAAM,aAAa,GAAG,+BAA+B,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;YAChE,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,QAAQ,EAAE,0BAA0B,EAAE,IAAI,CAAC,CAAC,CAAC;QACvF,CAAC;aAAM,IAAI,eAAe,EAAE,CAAC;YAC3B,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,eAAe,EAAE,QAAQ,EAAE,0BAA0B,EAAE,IAAI,CAAC,CAAC,CAAC;QACzF,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACb,oGAAoG,CACrG,CAAC;QACJ,CAAC;QAED,IAAI,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpD,cAAc,CACZ,uDAAuD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;gBACpF,6DAA6D,CAChE,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAEpD,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,yCAAyC,CAAC,CAAC;QAClF,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,iCAAiC,CAAC,CAAC;QAC5E,MAAM,EAAE,mBAAmB,EAAE,GAC3B,MAAM,MAAM,CAAC,gDAAgD,CAAC,CAAC;QAEjE,IAAI,EAAE,GAAG,KAAK,CAAC;QACf,IAAI,CAAC;YACH,CAAC,EAAE,EAAE,EAAE,GAAG,MAAM,aAAa,CAC3B,MAAM,EACN,mBAAmB,CAAC,UAAU,CAAC,EAC/B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAClB,UAAU,EACV,eAAe,EAAE,EACjB,MAAM;YACN,oFAAoF;YACpF,mBAAmB,CAAC,UAAU,EAAE,MAAM,CAAC,CACxC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACrE,EAAE,GAAG,KAAK,CAAC;QACb,CAAC;QAED,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,WAAW,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/commands/firstRunDialog.d.ts

@@ -1,4 +1,4 @@
-import { detectProviders, listModels, type DetectedProvider, type ModelInfo, type ProviderId } from '@gaunt-sloth/core/providers/modelDiscovery.js';
+import { detectProviders, type DetectedProvider, type ModelDiscoveryResult, type ModelInfo, type ProviderId } from '@gaunt-sloth/core/providers/modelDiscovery.js';
 import { ensureGslothDir, writeProjectReviewPreamble } from '#src/commands/configSetup.js';
 /**
  * Where the first-run dialog (CFG-2) persists the chosen provider/model config.
@@ -22,10 +22,29 @@ export type SelectFn = (title: string, options: string[], defaultIndex: number)
  */
 export interface FirstRunDialogDeps {
     detectProviders: typeof detectProviders;
-    listModels: typeof listModels;
+    /**
+     * CFG-21 — fetches the chosen provider's model list WITH its {@link ModelDiscoveryResult}
+     * provenance, baking in the generous interactive timeout by default. Returns `{ models, status }`
+     * so the dialog can surface an honest degrade notice when a live fetch was attempted and failed
+     * (`status: 'fallback'`) rather than silently swapping in a by-design curated list (`'curated'`).
+     */
+    fetchModels: (providerId: ProviderId) => Promise<ModelDiscoveryResult>;
+    /**
+     * CFG-21 — runs `run` while a visible "working" indicator is shown (an Ink spinner on an
+     * interactive TTY, a plain printed line otherwise), clearing it when `run` settles. The model
+     * fetch is routed through this seam so the wait is never silent and a test can assert it fired.
+     */
+    withProgress: <T>(label: string, run: () => Promise<T>) => Promise<T>;
     ensureGslothDir: typeof ensureGslothDir;
     writeProjectReviewPreamble: typeof writeProjectReviewPreamble;
-    /** Writes the config content to the scope's path and returns that path. */
+    /** Resolves the config file path for the chosen scope (does not write). */
+    resolveConfigPath: (scope: ConfigScope) => string;
+    /** True when a config file already exists at `path`. */
+    configExists: (path: string) => boolean;
+    /**
+     * (Over)writes the config content at the scope's path and returns that path. The dialog only
+     * calls this once it has decided to write, so this always writes (never a silent no-clobber skip).
+     */
     writeConfig: (scope: ConfigScope, content: string) => string;
     /** Prompts the user and resolves with their (untrimmed) answer. */
     ask: AskFn;
@@ -44,7 +63,14 @@ export declare function orderProviders(providers: DetectedProvider[]): DetectedP
 export declare function parseMenuSelection(answer: string, count: number): number | null;
 /** The index of the first ⭐ preferred model, or 0 when none are flagged. */
 export declare function defaultModelIndex(models: ModelInfo[]): number;
-/** Builds the `.gsloth.config.json` body for a chosen provider + model. */
+/**
+ * Builds the `.gsloth.config.json` body for a chosen provider + model. Delegates to the shared
+ * {@link buildInitConfigContent} so the interactive first-run path stamps the same `$schema`
+ * pointer (GS2-1) as the per-provider template writers. Previously this local builder omitted
+ * `$schema`, so a config created through the first-run dialog got no editor autocomplete/validation.
+ * First-run always has an explicit model chosen, so it is written into the config (unlike a bare
+ * `init <provider>` template, which omits the model to track the curated fallback, CFG-14).
+ */
 export declare function buildConfigContent(providerId: ProviderId, model: string): string;
 /** Resolves the config write path for the chosen scope (CFG-3 layering). */
 export declare function resolveConfigWritePath(scope: ConfigScope): string;
@@ -53,6 +79,14 @@ export declare function resolveConfigWritePath(scope: ConfigScope): string;
  * `✓`/blank mark read like a selection checkbox; this spells the state out instead.
  */
 export declare function providerReadinessLabel(provider: DetectedProvider): string;
+/**
+ * CFG-21 — the default {@link FirstRunDialogDeps.withProgress}: an Ink spinner while `run` is in
+ * flight when the environment can drive interactive Ink (the same activation policy as
+ * {@link makeDefaultSelect}), otherwise a plain printed line via {@link displayInfo} on the
+ * readline / non-TTY path. Either way the wait is visible, never silent. The Ink module is
+ * imported lazily so a readline-only run never loads React/Ink.
+ */
+export declare function makeDefaultProgress(): <T>(label: string, run: () => Promise<T>) => Promise<T>;
 /**
  * CFG-2 — the first-run configuration dialog.
  *
@@ -62,5 +96,5 @@ export declare function providerReadinessLabel(provider: DetectedProvider): stri
  * globally (CFG-3). Writes a minimal `{ llm: { type, model } }` config and, for
  * the project scope, scaffolds the review/guidelines preamble.
  */
-export declare function runFirstRunDialog(overrides?: Partial<FirstRunDialogDeps>): Promise<void>;
+export declare function runFirstRunDialog(overrides?: Partial<FirstRunDialogDeps>, force?: boolean): Promise<void>;
 export {};