@gaunt-sloth/batch 2.0.0-alpha.19 → 2.0.0-alpha.20

package/dist/pipelineCli.js

@@ -0,0 +1,348 @@
+/**
+ * @module pipelineCli
+ *
+ * BATCH-9 — the standalone `gth-batch` pipeline runner. A thin entry point that runs the BATCH-1
+ * matrix runtime ({@link buildMatrix} + {@link runBatchMatrix}) directly from a shell pipeline,
+ * without pulling in the whole `gaunt-sloth` app. It takes a prompt-executable script + `--over`
+ * data (inline JSON/YAML, or piped on stdin), runs the matrix, and streams the **same structured
+ * per-cell `CellResult`** that `gth batch` writes — one JSON object per line (JSONL) on stdout.
+ *
+ * Relationship to `gth batch` (packages/app/src/commands/batchCommand.ts): the two share the exact
+ * matrix runtime and the exact per-cell run wiring. The production per-cell adapter
+ * (`buildProductionRunCell`) lives in the app; this file mirrors it — its own `createResolvers()`,
+ * a lean agent factory, `runSingleShot` in `exec` mode, and `cleanupTools()` on every path — so the
+ * standalone bin needs only `@gaunt-sloth/core` + `@gaunt-sloth/agent` (both already dependencies),
+ * never the app. The only intentional differences: over-data is inline/stdin (not a file path, which
+ * the pipeline shell already handles) and output is JSONL on stdout (not a directory of files).
+ *
+ * stdout discipline: the run itself is noisy (the runtime's `display()`/`ProgressIndicator`/token
+ * streaming all target `process.stdout`). The bin entry ({@link file://./bin.ts}) redirects
+ * `process.stdout.write` to stderr for the duration and this module writes the machine JSONL
+ * straight to fd 1 (`fs.writeSync`), so stdout stays a clean data channel — the same "protocol
+ * channel" discipline `packages/app/cli.js` uses for ACP.
+ */
+import { writeSync } from 'node:fs';
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { parse as parseYaml } from 'yaml';
+import { initConfig as realInitConfig } from '@gaunt-sloth/core/config.js';
+import { buildSystemMessages, readExecPrompt, wrapContent, } from '@gaunt-sloth/core/utils/llmUtils.js';
+import { displayWarning } from '@gaunt-sloth/core/utils/consoleUtils.js';
+import { buildMatrix } from '#src/matrix.js';
+import { buildBatchSummary, runBatchMatrix } from '#src/BatchRunner.js';
+const USAGE = 'Usage: gth-batch <script> [--over <json|yaml>] [--models a,b,c] [-j <n>] [--retry <n>]\n' +
+    '  <script>        path to the .md prompt-executable to run over the matrix\n' +
+    '  --over <data>   inline JSON/YAML array of row objects (or pipe it on stdin)\n' +
+    '  --models a,b,c  comma-separated model axis (omit to use the configured model)\n' +
+    '  -j, --concurrency <n>  max in-flight cells\n' +
+    '  --retry <n>     retry a failed cell up to n times (default 0)';
+/** Parse `--models a,b,c` into a trimmed, non-empty list; `undefined` when empty/absent. */
+export function parseModels(models) {
+    const list = models
+        .split(',')
+        .map((m) => m.trim())
+        .filter((m) => m.length > 0);
+    return list.length > 0 ? list : undefined;
+}
+function parseIntArg(raw, flag) {
+    const n = Number.parseInt(raw, 10);
+    if (!Number.isFinite(n)) {
+        throw new Error(`${flag} expects an integer, got "${raw}"`);
+    }
+    return n;
+}
+/**
+ * Parse `gth-batch`'s argv (already sliced past `node <script>`). Pure and dependency-free so the
+ * arg surface is trivially unit-testable. Supports `--flag value` and `--flag=value`; throws a
+ * clear `Error` (with usage) on an unknown option, a missing value, or a missing `<script>`.
+ */
+export function parseArgs(argv) {
+    let script;
+    let over;
+    let models;
+    let concurrency;
+    let retry;
+    for (let i = 0; i < argv.length; i++) {
+        const arg = argv[i];
+        // Normalize `--flag=value` into a name + inline value.
+        let name = arg;
+        let inlineValue;
+        if (arg.startsWith('--') && arg.includes('=')) {
+            const eq = arg.indexOf('=');
+            name = arg.slice(0, eq);
+            inlineValue = arg.slice(eq + 1);
+        }
+        const takeValue = () => {
+            if (inlineValue !== undefined)
+                return inlineValue;
+            const v = argv[++i];
+            if (v === undefined)
+                throw new Error(`Missing value for ${name}\n\n${USAGE}`);
+            return v;
+        };
+        if (name === '--over') {
+            over = takeValue();
+        }
+        else if (name === '--models') {
+            models = parseModels(takeValue());
+        }
+        else if (name === '-j' || name === '--concurrency') {
+            concurrency = parseIntArg(takeValue(), name);
+        }
+        else if (name === '--retry') {
+            retry = parseIntArg(takeValue(), name);
+        }
+        else if (name === '-h' || name === '--help') {
+            throw new Error(USAGE);
+        }
+        else if (name.startsWith('-')) {
+            throw new Error(`Unknown option: ${arg}\n\n${USAGE}`);
+        }
+        else if (script === undefined) {
+            script = arg;
+        }
+        else {
+            throw new Error(`Unexpected extra argument: ${arg}\n\n${USAGE}`);
+        }
+    }
+    if (script === undefined) {
+        throw new Error(`Missing required <script> argument.\n\n${USAGE}`);
+    }
+    return { script, over, models, concurrency, retry };
+}
+/** Coerce a parsed row object's values to strings, mirroring `parseOver.ts`'s CSV/JSONL semantics
+ * (every field becomes a string that `{{field}}` interpolation binds). */
+function stringifyRowValues(record) {
+    const row = {};
+    for (const [key, value] of Object.entries(record)) {
+        row[key] = typeof value === 'string' ? value : JSON.stringify(value);
+    }
+    return row;
+}
+/**
+ * Parse inline/stdin `--over` data — a JSON or YAML **array of row objects** — into
+ * {@link MatrixRow}s. (The app's `parseOverFile` picks CSV vs JSONL by file extension; a shell
+ * pipeline has no filename, so this bin standardizes on one self-describing format: JSON, which is
+ * also valid YAML.) Throws a descriptive `Error` on malformed input — the one thing that should
+ * make the bin exit non-zero.
+ */
+export function parseOverData(text) {
+    let parsed;
+    try {
+        parsed = parseYaml(text);
+    }
+    catch (error) {
+        throw new Error(`--over data is not valid JSON/YAML: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    if (!Array.isArray(parsed)) {
+        throw new Error('--over data must be a JSON/YAML array of row objects');
+    }
+    if (parsed.length === 0) {
+        throw new Error('--over data is an empty array (no rows)');
+    }
+    return parsed.map((row, i) => {
+        if (row === null || typeof row !== 'object' || Array.isArray(row)) {
+            throw new Error(`--over row ${i} must be an object, got ${JSON.stringify(row)}`);
+        }
+        return stringifyRowValues(row);
+    });
+}
+/**
+ * Resolve the input axis: inline `--over` if present, otherwise piped stdin, otherwise `undefined`
+ * (no input axis → a single cell over the script's own content, exactly like `exec`).
+ */
+export async function resolveRows(args, readStdin) {
+    if (args.over !== undefined) {
+        return parseOverData(args.over);
+    }
+    const piped = (await readStdin()).trim();
+    if (piped.length > 0) {
+        return parseOverData(piped);
+    }
+    return undefined;
+}
+/** Build the exec-mode system preamble the same way `gth batch` does (app's `getExecSystemPrompt`),
+ * flattened to a string, using only `@gaunt-sloth/core` prompt builders. */
+export function getExecPreamble(config) {
+    const [systemMessage] = buildSystemMessages(config, readExecPrompt(config));
+    const content = systemMessage?.content;
+    if (typeof content === 'string') {
+        return content;
+    }
+    if (Array.isArray(content)) {
+        return content
+            .map((item) => (typeof item === 'string' ? item : 'text' in item ? item.text : ''))
+            .join('\n');
+    }
+    return '';
+}
+/**
+ * Build a per-`model` {@link GthConfig} cache — the same idiom as `createCellConfigResolver`
+ * (batchCommand.ts) / `createModelConfigResolver` (runWorkflow.ts): `undefined` model → the base
+ * config; a named model → one cached `initConfig({ ...overrides, model })` call (cached by Promise
+ * so concurrent cells for the same not-yet-resolved model share one in-flight build).
+ */
+function createModelConfigResolver(baseConfig, overrides, initConfig) {
+    const configForModel = new Map();
+    return (model) => {
+        if (!model)
+            return Promise.resolve(baseConfig);
+        let cached = configForModel.get(model);
+        if (!cached) {
+            cached = initConfig({ ...overrides, model });
+            configForModel.set(model, cached);
+        }
+        return cached;
+    };
+}
+/**
+ * The production per-cell adapter, mirroring `buildProductionRunCell` (batchCommand.ts) exactly:
+ * a fresh `createResolvers()` per cell, a lean agent factory, `runSingleShot` in `exec` mode, and
+ * `cleanupTools()` on every path. Kept here (not imported from the app) so the standalone bin
+ * depends only on core + agent.
+ */
+async function buildProductionRunCell(baseConfig, preamble, overrides, initConfig) {
+    const { runSingleShot } = await import('@gaunt-sloth/core/runtime/singleShot.js');
+    const { createResolvers } = await import('@gaunt-sloth/agent/resolvers.js');
+    const { resolveAgentFactory } = await import('@gaunt-sloth/agent/core/resolveAgentFactory.js');
+    const resolveCellModelConfig = createModelConfigResolver(baseConfig, overrides, initConfig);
+    return async (cell) => {
+        const modelConfig = await resolveCellModelConfig(cell.model);
+        const cellConfig = {
+            ...modelConfig,
+            canInterruptInferenceWithEsc: false,
+            writeOutputToFile: false,
+            // Non-interactive pipeline: don't stream tokens to the (redirected) stdout; the answer is
+            // captured in the cell's JSON record instead.
+            streamOutput: false,
+        };
+        const content = wrapContent(cell.content, 'script', 'prompt-executable script', true);
+        const resolvers = createResolvers();
+        try {
+            const { ok, answer, tokensInput, tokensOutput, tools } = await runSingleShot(`BATCH-${cell.id}`, preamble, content, cellConfig, resolvers, 'exec', resolveAgentFactory(cellConfig, 'lean'));
+            return { ok, answer, tokensInput, tokensOutput, tools };
+        }
+        catch (error) {
+            // runSingleShot is documented to not throw for a normal LLM/tool failure; this guards the
+            // rare genuinely-unexpected exception so one bad cell never takes the whole batch down.
+            return { ok: false, error: error instanceof Error ? error.message : String(error) };
+        }
+        finally {
+            try {
+                await resolvers.cleanupTools?.();
+            }
+            catch (cleanupError) {
+                displayWarning(`Failed to clean up tools for cell ${cell.id}: ` +
+                    `${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`);
+            }
+        }
+    };
+}
+/**
+ * Run the matrix and stream one JSON `CellResult` per line (JSONL) to `write`. Returns the
+ * aggregate {@link BatchSummary}. Each emitted line is the exact object `gth batch` writes to
+ * `<id>.json` (compact rather than pretty-printed) — the "same structured per-cell output".
+ */
+export async function runMatrixToStream(scriptContent, models, rows, options) {
+    const cells = buildMatrix(scriptContent, models, rows);
+    const results = await runBatchMatrix(cells, {
+        runCell: options.runCell,
+        concurrency: options.concurrency,
+        retry: options.retry,
+    });
+    for (const result of results) {
+        options.write(`${JSON.stringify(result)}\n`);
+    }
+    return buildBatchSummary(results);
+}
+async function readStdinToEnd() {
+    const stdin = process.stdin;
+    if (stdin.isTTY)
+        return '';
+    const chunks = [];
+    for await (const chunk of stdin) {
+        chunks.push(Buffer.from(chunk));
+    }
+    return Buffer.concat(chunks).toString('utf8');
+}
+function readScriptFile(scriptPath) {
+    try {
+        return readFileSync(resolve(scriptPath), 'utf8');
+    }
+    catch (error) {
+        throw new Error(`Cannot read script "${scriptPath}": ${error instanceof Error ? error.message : String(error)}`);
+    }
+}
+/** A broken-pipe error — a downstream reader (`head`, `jq 'first'`, a quit pager) closed the pipe. */
+function isEpipe(error) {
+    return !!error && typeof error === 'object' && error.code === 'EPIPE';
+}
+/**
+ * The standalone `gth-batch` entry. Parses argv, resolves the script + input axis, wires the
+ * per-cell runtime, runs the matrix, and streams JSONL cell records via `deps.write`.
+ *
+ * Exit-code contract (identical to `gth batch`): resolves to `0` iff the cells *ran* — a per-cell
+ * failure is recorded in that cell's JSON, never reflected in the exit code. A harness-level error
+ * (bad args, unreadable script, malformed `--over`, config failure) resolves to `1`. A downstream
+ * reader closing the pipe early (`EPIPE` on the stdout channel — e.g. `| head`) is normal pipeline
+ * usage and also resolves to `0`: the partial output already written is correct.
+ *
+ * @returns the process exit code (0 or 1).
+ */
+export async function runBatchCli(argv, deps = {}) {
+    // Default stdout sink: once a downstream reader closes the pipe (EPIPE), stop writing and treat
+    // it as a clean stop rather than letting the broken-pipe error bubble up as a harness failure.
+    let stdoutClosed = false;
+    const write = deps.write ??
+        ((chunk) => {
+            if (stdoutClosed)
+                return;
+            try {
+                writeSync(1, chunk);
+            }
+            catch (error) {
+                if (isEpipe(error)) {
+                    stdoutClosed = true;
+                    return;
+                }
+                throw error;
+            }
+        });
+    const logError = deps.logError ?? ((chunk) => void process.stderr.write(chunk));
+    try {
+        const args = parseArgs(argv);
+        const readScript = deps.readScript ?? readScriptFile;
+        const readStdin = deps.readStdin ?? readStdinToEnd;
+        const initConfig = deps.initConfig ?? realInitConfig;
+        const scriptContent = readScript(args.script);
+        const rows = await resolveRows(args, readStdin);
+        let runCell = deps.runCell;
+        if (!runCell) {
+            const overrides = {};
+            const baseConfig = await initConfig(overrides);
+            const preamble = getExecPreamble(baseConfig);
+            runCell = await buildProductionRunCell(baseConfig, preamble, overrides, initConfig);
+        }
+        const summary = await runMatrixToStream(scriptContent, args.models, rows, {
+            concurrency: args.concurrency,
+            retry: args.retry,
+            runCell,
+            write,
+        });
+        logError(`gth-batch: ${summary.passed}/${summary.total} cell(s) ok` +
+            (summary.failed > 0 ? `, ${summary.failed} failed` : '') +
+            '\n');
+        return 0;
+    }
+    catch (error) {
+        // A broken pipe on the stdout data channel (a downstream reader closed early) is a clean stop,
+        // not a harness error: exit 0 and stay silent. Genuine harness errors (bad args, unreadable
+        // script, malformed --over, config failure) carry no EPIPE code and still exit non-zero.
+        if (isEpipe(error)) {
+            return 0;
+        }
+        logError(`gth-batch: ${error instanceof Error ? error.message : String(error)}\n`);
+        return 1;
+    }
+}
+//# sourceMappingURL=pipelineCli.js.map

package/dist/pipelineCli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipelineCli.js","sourceRoot":"","sources":["../src/pipelineCli.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAE1C,OAAO,EAAE,UAAU,IAAI,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAE3E,OAAO,EACL,mBAAmB,EACnB,cAAc,EACd,WAAW,GACZ,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,yCAAyC,CAAC;AAEzE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAkCxE,MAAM,KAAK,GACT,0FAA0F;IAC1F,8EAA8E;IAC9E,iFAAiF;IACjF,mFAAmF;IACnF,gDAAgD;IAChD,iEAAiE,CAAC;AAEpE,4FAA4F;AAC5F,MAAM,UAAU,WAAW,CAAC,MAAc;IACxC,MAAM,IAAI,GAAG,MAAM;SAChB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC/B,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AAC5C,CAAC;AAED,SAAS,WAAW,CAAC,GAAW,EAAE,IAAY;IAC5C,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACnC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,6BAA6B,GAAG,GAAG,CAAC,CAAC;IAC9D,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,SAAS,CAAC,IAAc;IACtC,IAAI,MAA0B,CAAC;IAC/B,IAAI,IAAwB,CAAC;IAC7B,IAAI,MAA4B,CAAC;IACjC,IAAI,WAA+B,CAAC;IACpC,IAAI,KAAyB,CAAC;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAEpB,uDAAuD;QACvD,IAAI,IAAI,GAAG,GAAG,CAAC;QACf,IAAI,WAA+B,CAAC;QACpC,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9C,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACxB,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,SAAS,GAAG,GAAW,EAAE;YAC7B,IAAI,WAAW,KAAK,SAAS;gBAAE,OAAO,WAAW,CAAC;YAClD,MAAM,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YACpB,IAAI,CAAC,KAAK,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,OAAO,KAAK,EAAE,CAAC,CAAC;YAC9E,OAAO,CAAC,CAAC;QACX,CAAC,CAAC;QAEF,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtB,IAAI,GAAG,SAAS,EAAE,CAAC;QACrB,CAAC;aAAM,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;YAC/B,MAAM,GAAG,WAAW,CAAC,SAAS,EAAE,CAAC,CAAC;QACpC,CAAC;aAAM,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,eAAe,EAAE,CAAC;YACrD,WAAW,GAAG,WAAW,CAAC,SAAS,EAAE,EAAE,IAAI,CAAC,CAAC;QAC/C,CAAC;aAAM,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YAC9B,KAAK,GAAG,WAAW,CAAC,SAAS,EAAE,EAAE,IAAI,CAAC,CAAC;QACzC,CAAC;aAAM,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;aAAM,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,OAAO,KAAK,EAAE,CAAC,CAAC;QACxD,CAAC;aAAM,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YAChC,MAAM,GAAG,GAAG,CAAC;QACf,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,8BAA8B,GAAG,OAAO,KAAK,EAAE,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IAED,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,0CAA0C,KAAK,EAAE,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;AACtD,CAAC;AAED;0EAC0E;AAC1E,SAAS,kBAAkB,CAAC,MAA+B;IACzD,MAAM,GAAG,GAAc,EAAE,CAAC;IAC1B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAClD,GAAG,CAAC,GAAG,CAAC,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACvE,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,uCAAuC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAChG,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;QAC3B,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAClE,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,2BAA2B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACnF,CAAC;QACD,OAAO,kBAAkB,CAAC,GAA8B,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAkB,EAClB,SAAgC;IAEhC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC5B,OAAO,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACzC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC;IAC9B,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;4EAC4E;AAC5E,MAAM,UAAU,eAAe,CAAC,MAAiB;IAC/C,MAAM,CAAC,aAAa,CAAC,GAAG,mBAAmB,CAAC,MAAM,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5E,MAAM,OAAO,GAAG,aAAa,EAAE,OAAO,CAAC;IACvC,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,OAAO,OAAO;aACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;aAClF,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;;;;GAKG;AACH,SAAS,yBAAyB,CAChC,UAAqB,EACrB,SAAqC,EACrC,UAAyE;IAEzE,MAAM,cAAc,GAAG,IAAI,GAAG,EAA8B,CAAC;IAC7D,OAAO,CAAC,KAAyB,EAAsB,EAAE;QACvD,IAAI,CAAC,KAAK;YAAE,OAAO,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC/C,IAAI,MAAM,GAAG,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,GAAG,UAAU,CAAC,EAAE,GAAG,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;YAC7C,cAAc,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,sBAAsB,CACnC,UAAqB,EACrB,QAAgB,EAChB,SAAqC,EACrC,UAAyE;IAEzE,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,yCAAyC,CAAC,CAAC;IAClF,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,iCAAiC,CAAC,CAAC;IAC5E,MAAM,EAAE,mBAAmB,EAAE,GAAG,MAAM,MAAM,CAAC,gDAAgD,CAAC,CAAC;IAE/F,MAAM,sBAAsB,GAAG,yBAAyB,CAAC,UAAU,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;IAE5F,OAAO,KAAK,EAAE,IAAI,EAAE,EAAE;QACpB,MAAM,WAAW,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7D,MAAM,UAAU,GAAc;YAC5B,GAAG,WAAW;YACd,4BAA4B,EAAE,KAAK;YACnC,iBAAiB,EAAE,KAAK;YACxB,0FAA0F;YAC1F,8CAA8C;YAC9C,YAAY,EAAE,KAAK;SACpB,CAAC;QACF,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,0BAA0B,EAAE,IAAI,CAAC,CAAC;QAEtF,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,IAAI,CAAC;YACH,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,MAAM,aAAa,CAC1E,SAAS,IAAI,CAAC,EAAE,EAAE,EAClB,QAAQ,EACR,OAAO,EACP,UAAU,EACV,SAAS,EACT,MAAM,EACN,mBAAmB,CAAC,UAAU,EAAE,MAAM,CAAC,CACxC,CAAC;YACF,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;QAC1D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,0FAA0F;YAC1F,wFAAwF;YACxF,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACtF,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC;gBACH,MAAM,SAAS,CAAC,YAAY,EAAE,EAAE,CAAC;YACnC,CAAC;YAAC,OAAO,YAAY,EAAE,CAAC;gBACtB,cAAc,CACZ,qCAAqC,IAAI,CAAC,EAAE,IAAI;oBAC9C,GAAG,YAAY,YAAY,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CACnF,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,aAAqB,EACrB,MAA4B,EAC5B,IAA6B,EAC7B,OAKC;IAED,MAAM,KAAK,GAAG,WAAW,CAAC,aAAa,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;IACvD,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,KAAK,EAAE;QAC1C,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,KAAK,EAAE,OAAO,CAAC,KAAK;KACrB,CAAC,CAAC;IACH,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;AACpC,CAAC;AAED,KAAK,UAAU,cAAc;IAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;IAC5B,IAAI,KAAK,CAAC,KAAK;QAAE,OAAO,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAChC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,cAAc,CAAC,UAAkB;IACxC,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC;IACnD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,uBAAuB,UAAU,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAChG,CAAC;IACJ,CAAC;AACH,CAAC;AAED,sGAAsG;AACtG,SAAS,OAAO,CAAC,KAAc;IAC7B,OAAO,CAAC,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAK,KAA+B,CAAC,IAAI,KAAK,OAAO,CAAC;AACnG,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAc,EAAE,IAAI,GAAiB,EAAE;IACvE,gGAAgG;IAChG,+FAA+F;IAC/F,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,MAAM,KAAK,GACT,IAAI,CAAC,KAAK;QACV,CAAC,CAAC,KAAa,EAAQ,EAAE;YACvB,IAAI,YAAY;gBAAE,OAAO;YACzB,IAAI,CAAC;gBACH,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACtB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;oBACnB,YAAY,GAAG,IAAI,CAAC;oBACpB,OAAO;gBACT,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC,CAAC,CAAC;IACL,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAa,EAAQ,EAAE,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IAE9F,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,cAAc,CAAC;QACrD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,cAAc,CAAC;QACnD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,cAAc,CAAC;QAErD,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAEhD,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,SAAS,GAA+B,EAAE,CAAC;YACjD,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,CAAC;YAC/C,MAAM,QAAQ,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;YAC7C,OAAO,GAAG,MAAM,sBAAsB,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;QACtF,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,aAAa,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE;YACxE,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,OAAO;YACP,KAAK;SACN,CAAC,CAAC;QAEH,QAAQ,CACN,cAAc,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,aAAa;YACxD,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,MAAM,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;YACxD,IAAI,CACP,CAAC;QACF,OAAO,CAAC,CAAC;IACX,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,+FAA+F;QAC/F,4FAA4F;QAC5F,yFAAyF;QACzF,IAAI,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACnB,OAAO,CAAC,CAAC;QACX,CAAC;QACD,QAAQ,CAAC,cAAc,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACnF,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC"}

package/dist/toolChecks.d.ts

@@ -0,0 +1,21 @@
+import type { EvalExpectation } from '#src/evalTypes.js';
+/**
+ * BATCH-10 tool-trace assertions — grade a case against the tool *names* it actually invoked
+ * (`cellResult.tools`), rather than grepping the answer text. This "kills the false-positive class
+ * structurally": an MCP-only case can prove the server was called instead of a substring that a
+ * hallucinated answer could also contain.
+ *
+ * Kept separate from `runDeterministicChecks` (which reads the *answer*) on purpose — these read the
+ * tool trace — so neither signature is muddied. Patterns reuse GS2-61's `toolNameMatchesPattern`
+ * (`@gaunt-sloth/core`), the same glob/exact matcher `allowedTools` uses, so `mcp__unimarket__*`
+ * behaves identically here.
+ *
+ * - `mustCall` — for **each** pattern, at least one called tool must match it, else
+ *   `did not call "<pattern>"`.
+ * - `mustNotCall` — **no** called tool may match any forbidden pattern; each offending tool is
+ *   reported once as `called forbidden tool "<tool>" (matched "<pattern>")`.
+ *
+ * BATCH-12: grades one {@link EvalExpectation} block's tool-trace assertions (a flat case's single
+ * block or a matrix case's identity-scoped block) — same field names, same behavior as before.
+ */
+export declare function runToolCallChecks(tools: string[], expectation: Pick<EvalExpectation, 'mustCall' | 'mustNotCall'>): string[];

package/dist/toolChecks.js

@@ -0,0 +1,38 @@
+import { toolNameMatchesPattern } from '@gaunt-sloth/core/utils/toolMatching.js';
+/**
+ * BATCH-10 tool-trace assertions — grade a case against the tool *names* it actually invoked
+ * (`cellResult.tools`), rather than grepping the answer text. This "kills the false-positive class
+ * structurally": an MCP-only case can prove the server was called instead of a substring that a
+ * hallucinated answer could also contain.
+ *
+ * Kept separate from `runDeterministicChecks` (which reads the *answer*) on purpose — these read the
+ * tool trace — so neither signature is muddied. Patterns reuse GS2-61's `toolNameMatchesPattern`
+ * (`@gaunt-sloth/core`), the same glob/exact matcher `allowedTools` uses, so `mcp__unimarket__*`
+ * behaves identically here.
+ *
+ * - `mustCall` — for **each** pattern, at least one called tool must match it, else
+ *   `did not call "<pattern>"`.
+ * - `mustNotCall` — **no** called tool may match any forbidden pattern; each offending tool is
+ *   reported once as `called forbidden tool "<tool>" (matched "<pattern>")`.
+ *
+ * BATCH-12: grades one {@link EvalExpectation} block's tool-trace assertions (a flat case's single
+ * block or a matrix case's identity-scoped block) — same field names, same behavior as before.
+ */
+export function runToolCallChecks(tools, expectation) {
+    const failures = [];
+    for (const pattern of expectation.mustCall) {
+        if (!tools.some((tool) => toolNameMatchesPattern(tool, pattern))) {
+            failures.push(`did not call "${pattern}"`);
+        }
+    }
+    // De-duplicate: a tool called N times (traces realistically repeat names) is one violation, not
+    // N identical failure lines. `Set` preserves first-seen order.
+    for (const tool of new Set(tools)) {
+        const matched = expectation.mustNotCall.find((pattern) => toolNameMatchesPattern(tool, pattern));
+        if (matched !== undefined) {
+            failures.push(`called forbidden tool "${tool}" (matched "${matched}")`);
+        }
+    }
+    return failures;
+}
+//# sourceMappingURL=toolChecks.js.map

package/dist/toolChecks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"toolChecks.js","sourceRoot":"","sources":["../src/toolChecks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AAIjF;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,iBAAiB,CAC/B,KAAe,EACf,WAA8D;IAE9D,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,KAAK,MAAM,OAAO,IAAI,WAAW,CAAC,QAAQ,EAAE,CAAC;QAC3C,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,EAAE,CAAC;YACjE,QAAQ,CAAC,IAAI,CAAC,iBAAiB,OAAO,GAAG,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,gGAAgG;IAChG,+DAA+D;IAC/D,KAAK,MAAM,IAAI,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CACvD,sBAAsB,CAAC,IAAI,EAAE,OAAO,CAAC,CACtC,CAAC;QACF,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,0BAA0B,IAAI,eAAe,OAAO,IAAI,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gaunt-sloth/batch",
-  "version": "2.0.0-alpha.19",
+  "version": "2.0.0-alpha.20",
   "description": "Batch matrix runtime for Gaunt Sloth (BATCH-1): run a prompt-executable over a matrix of models and/or content-bound inputs",
   "license": "MIT",
   "author": "Andrew Kondratev",
@@ -23,6 +23,9 @@
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
+  "bin": {
+    "gth-batch": "dist/bin.js"
+  },
   "exports": {
     ".": "./dist/index.js",
     "./*.js": "./dist/*.js"
@@ -34,8 +37,8 @@
     "@langchain/core": "^1.2.3",
     "yaml": "^2.9.0",
     "zod": "^4.4.3",
-    "@gaunt-sloth/agent": "2.0.0-alpha.19",
-    "@gaunt-sloth/core": "2.0.0-alpha.19"
+    "@gaunt-sloth/agent": "2.0.0-alpha.20",
+    "@gaunt-sloth/core": "2.0.0-alpha.20"
   },
   "files": [
     "./dist/*"