@openagentsy/hooks 0.0.0-stage → 0.2.0

package/dist/eval/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAQA,OAAO,EACL,YAAY,EACZ,KAAK,cAAc,EACnB,KAAK,UAAU,EACf,KAAK,YAAY,EAGlB,MAAM,YAAY,CAAC;AAqDpB,qEAAqE;AACrE,wBAAgB,QAAQ,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAOrF;AAED;;;;;GAKG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,MAAM,EAAE,CA2BxE;AAED,0EAA0E;AAC1E,wBAAgB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE3C;AAED,2BAA2B;AAC3B,wBAAgB,WAAW,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAE9D;AAED,2BAA2B;AAC3B,MAAM,MAAM,UAAU,GAAG;IACvB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAqBF;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAwCjG;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,UAAU,GAAG,cAAc,CAqBxF;AAED,oCAAoC;AACpC,wBAAgB,SAAS,CAAC,UAAU,EAAE,cAAc,GAAG,OAAO,CAE7D;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAExD;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,GAAG,MAAM,CAiBpF;AAoCD;;;;;;;;GAQG;AACH,wBAAsB,eAAe,CACnC,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,UAAU,EAClB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,CAuCjB;AAcD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI,CAqBzD;AAED,wBAAwB;AACxB,MAAM,MAAM,cAAc,GAAG;IAC3B,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,+CAA+C;IAC/C,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,mEAAmE;IACnE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2EAA2E;IAC3E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+CAA+C;IAC/C,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,cAAc,GAAG,MAAM,EAAE,CAG9D;AAaD;;;;;;;;;;GAUG;AACH,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,YAAY,EACtB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EAAE,GACd,OAAO,CAAC,MAAM,CAAC,CAyBjB;AA4BD;;;;;GAKG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAWtE;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC,CA6DlF;AAgBD;;;;;GAKG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAE5D;AAgDD;;;;;;;;;;;GAWG;AACH,wBAAsB,qBAAqB,CAAC,CAAC,EAAE,OAAO,EAAE;IACtD,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;CACvC,GAAG,OAAO,CAAC,CAAC,CAAC,CAiCb;AAED,mCAAmC;AACnC,MAAM,MAAM,eAAe,GAAG;IAC5B,iEAAiE;IACjE,eAAe,EAAE,OAAO,CAAC;IACzB,gEAAgE;IAChE,kBAAkB,EAAE,OAAO,CAAC;IAC5B,+BAA+B;IAC/B,MAAM,EAAE,OAAO,CAAC;IAChB,mEAAmE;IACnE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uDAAuD;IACvD,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,kEAAkE;IAClE,KAAK,CAAC,EAAE,UAAU,CAAC;CACpB,CAAC;AAEF;;;;;;GAMG;AACH,wBAAsB,iBAAiB,CAAC,OAAO,EAAE;IAC/C,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB,GAAG,OAAO,CAAC,eAAe,CAAC,CA+C3B"}

package/dist/eval/runner.js

@@ -0,0 +1,616 @@
+import { spawn } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { constants } from 'node:fs';
+import { chmod, lstat, mkdir, mkdtemp, open, readFile, realpath, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { dirname, isAbsolute, join, relative, resolve } from 'node:path';
+import { isLessonId } from '../lessons/id.js';
+import { isInside, projectName, worktreeRoot } from '../paths.js';
+import { EvalArtifact, ProofError, } from './types.js';
+/**
+ * The eval runner.
+ *
+ * A lesson is only worth applying when something shows it works. The proof
+ * is an artifact: a list of scenarios, each a command and what it must show.
+ * The runner executes them under four rules, and every one of them is a
+ * boundary rather than a preference:
+ *
+ * - the command is an argv, spawned without a shell;
+ * - the child sees only PATH, HOME, TMPDIR and the locale, so a token in the
+ *   parent's environment never reaches a proof;
+ * - the working directory is inside the workspace or an OpenAgentsy worktree,
+ *   and it is the real path that has to be inside it;
+ * - the process is killed at the timeout.
+ *
+ * The result holds the verdicts and the hashes. It never holds what a
+ * scenario printed, because a result is kept and a printout is not.
+ */
+/**
+ * The environment a proof may see.
+ *
+ * The list is an allowlist, not a denylist: a variable that is not named
+ * here is not in the child's environment at all, so a token that a session
+ * happened to hold cannot leak into a proof.
+ */
+const ENV_ALLOWLIST = [
+    'PATH',
+    'HOME',
+    'TMPDIR',
+    'LANG',
+    'LANGUAGE',
+    'LC_ALL',
+    'LC_CTYPE',
+    'LC_COLLATE',
+    'LC_MESSAGES',
+    'LC_NUMERIC',
+    'LC_TIME',
+];
+/**
+ * The characters that make a string a shell string.
+ *
+ * A command that is one element and holds one of these is a shell line that
+ * was wrapped in an array, and the runner refuses it rather than run it.
+ */
+const SHELL_STRING = /[\s;&|<>`$(){}[\]!*?"'\\]/;
+/** The most of a run's output the runner keeps, per stream. */
+const MAX_CAPTURE_BYTES = 1024 * 1024;
+/** The environment a proof gets: the allowlist, and nothing else. */
+export function childEnv(env = process.env) {
+    const out = {};
+    for (const key of ENV_ALLOWLIST) {
+        const value = env[key];
+        if (value !== undefined)
+            out[key] = value;
+    }
+    return out;
+}
+/**
+ * The command, checked.
+ *
+ * The schema already holds an array, and this says why: a string is a shell
+ * line, and a shell line is a program the runner does not choose.
+ */
+export function assertArgv(command) {
+    if (!Array.isArray(command) || command.length === 0) {
+        throw new ProofError('the command must be an argv array', 'write the program and its arguments as separate elements, never as one shell string');
+    }
+    for (const part of command) {
+        if (typeof part !== 'string' || part.length === 0) {
+            throw new ProofError('every element of the command must be a non-empty string', 'write the program and its arguments as separate elements, never as one shell string');
+        }
+        if (part.includes('\n')) {
+            throw new ProofError('an argument holds a newline', 'a newline in an argument is a shell line in disguise; pass the argument as one element without it');
+        }
+    }
+    if (command.length === 1 && SHELL_STRING.test(command[0])) {
+        throw new ProofError('the command looks like a shell string', 'write the program and its arguments as separate elements, never as one shell string');
+    }
+}
+/** The hash of a command, so a result can name it without carrying it. */
+export function hashOf(text) {
+    return createHash('sha256').update(text).digest('hex');
+}
+/** The hash of an argv. */
+export function commandHash(command) {
+    return hashOf(JSON.stringify(command));
+}
+/**
+ * Kill the process and everything it started.
+ *
+ * The child is its own process group, so the group is what dies: a proof that
+ * starts a server must not leave the server behind at the timeout.
+ */
+function killGroup(child) {
+    const pid = child.pid;
+    if (pid === undefined) {
+        child.kill('SIGKILL');
+        return;
+    }
+    try {
+        process.kill(-pid, 'SIGKILL');
+    }
+    catch {
+        child.kill('SIGKILL');
+    }
+}
+/**
+ * Run one command.
+ *
+ * The spawn has no shell, the environment is the allowlist, and the timeout
+ * kills the group. The output is kept in memory for the assertions and is
+ * never written anywhere.
+ */
+export function runCommand(command, cwd, timeoutMs) {
+    assertArgv(command);
+    return new Promise((resolveOutcome) => {
+        const child = spawn(command[0], command.slice(1), {
+            cwd,
+            env: childEnv(),
+            shell: false,
+            detached: true,
+            stdio: ['ignore', 'pipe', 'pipe'],
+        });
+        let stdout = '';
+        let stderr = '';
+        let timedOut = false;
+        let settled = false;
+        const timer = setTimeout(() => {
+            timedOut = true;
+            killGroup(child);
+        }, timeoutMs);
+        const finish = (outcome) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            resolveOutcome(outcome);
+        };
+        child.stdout?.on('data', (chunk) => {
+            if (stdout.length < MAX_CAPTURE_BYTES)
+                stdout += chunk.toString('utf8');
+        });
+        child.stderr?.on('data', (chunk) => {
+            if (stderr.length < MAX_CAPTURE_BYTES)
+                stderr += chunk.toString('utf8');
+        });
+        child.on('error', (error) => {
+            finish({ exitCode: null, timedOut, stdout, stderr: `${stderr}${error.message}` });
+        });
+        child.on('close', (code) => {
+            finish({ exitCode: code, timedOut, stdout, stderr });
+        });
+    });
+}
+/**
+ * The verdicts of one scenario.
+ *
+ * A scenario that only names an exit code does not care what was printed, so
+ * only the named assertions are recorded. The spawn itself is always one:
+ * a command that could not start has not passed.
+ */
+export function assertionsOf(scenario, outcome) {
+    const assertions = {
+        spawned: outcome.exitCode !== null || outcome.timedOut,
+    };
+    const expect = scenario.expect;
+    if (expect.exit_code !== undefined) {
+        assertions['exit_code'] = outcome.exitCode === expect.exit_code;
+    }
+    (expect.stdout_includes ?? []).forEach((needle, index) => {
+        assertions[`stdout_includes[${index}]`] = outcome.stdout.includes(needle);
+    });
+    (expect.stdout_excludes ?? []).forEach((needle, index) => {
+        assertions[`stdout_excludes[${index}]`] = !outcome.stdout.includes(needle);
+    });
+    (expect.stderr_includes ?? []).forEach((needle, index) => {
+        assertions[`stderr_includes[${index}]`] = outcome.stderr.includes(needle);
+    });
+    (expect.stderr_excludes ?? []).forEach((needle, index) => {
+        assertions[`stderr_excludes[${index}]`] = !outcome.stderr.includes(needle);
+    });
+    return assertions;
+}
+/** Whether every assertion held. */
+export function allPassed(assertions) {
+    return Object.values(assertions).every((value) => value === true);
+}
+/**
+ * The directory the results live in.
+ */
+export function evalResultsDir(stateRoot) {
+    return join(stateRoot, '.openagentsy', 'lessons', 'evals');
+}
+/**
+ * The path a result is written to.
+ *
+ * The lesson id names the file, so it is checked before it names anything:
+ * an id that is really a path would write the result outside the directory
+ * the results belong in.
+ */
+export function evalResultPath(stateRoot, lessonId, at) {
+    if (!isLessonId(lessonId)) {
+        throw new ProofError(`"${lessonId}" is not a lesson id`, 'a lesson id looks like oa-lesson-abcd1234');
+    }
+    const dir = evalResultsDir(stateRoot);
+    const stamp = at.toISOString().replace(/[:.]/g, '-');
+    const path = join(dir, `${lessonId}-${stamp}.json`);
+    if (!isInside(path, [dir])) {
+        throw new ProofError('the result path leaves the evaluations directory', 'a result belongs to the lesson it is about, and the lesson id is not a path');
+    }
+    return path;
+}
+/**
+ * Refuse a state path that is a symlink.
+ *
+ * The result is written under the state root, and every part of that path
+ * has to be a real directory: a symlinked `.openagentsy` or `lessons` would
+ * put the result somewhere else entirely, and the runner cannot know where.
+ */
+async function assertRealStatePath(stateRoot, dir) {
+    const rest = relative(stateRoot, dir);
+    if (rest.startsWith('..')) {
+        throw new ProofError('the results directory is not under the state root', 'a result belongs to the project it was run in');
+    }
+    const realRoot = await realpath(stateRoot);
+    let current = realRoot;
+    for (const segment of rest.split('/').filter((part) => part.length > 0)) {
+        current = join(current, segment);
+        let info;
+        try {
+            info = await lstat(current);
+        }
+        catch {
+            return;
+        }
+        if (info.isSymbolicLink()) {
+            throw new ProofError(`the state path ${current} is a symlink`, 'the runner refuses to write a result through a symlink');
+        }
+    }
+}
+/**
+ * Write a result.
+ *
+ * The mode is 0600, because a result says what a proof did in a project and
+ * nobody else has a reason to read it. The file is opened with
+ * O_NOFOLLOW, so a symlink at the path is not followed, and with O_EXCL, so
+ * an existing file is never replaced: a result is evidence and evidence is
+ * not overwritten.
+ */
+export async function writeEvalResult(path, result, stateRoot) {
+    const dir = dirname(path);
+    await assertRealStatePath(stateRoot, dir);
+    await mkdir(dir, { recursive: true });
+    const realRoot = await realpath(stateRoot);
+    const realDir = await realpath(dir);
+    if (!isInside(realDir, [join(realRoot, '.openagentsy')])) {
+        throw new ProofError('the results directory is not inside the state root', 'the runner refuses to write a result outside the state root');
+    }
+    let target = path;
+    let counter = 1;
+    while (await exists(target)) {
+        const info = await lstat(target);
+        if (info.isSymbolicLink()) {
+            throw new ProofError('the result path is a symlink', 'the runner refuses to write a result through a symlink');
+        }
+        counter += 1;
+        target = path.replace(/\.json$/, '').concat(`-${counter}.json`);
+    }
+    const handle = await open(target, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, 0o600);
+    try {
+        await handle.writeFile(`${JSON.stringify(result, null, 2)}\n`, 'utf8');
+    }
+    finally {
+        await handle.close();
+    }
+    await chmod(target, 0o600);
+    return target;
+}
+/**
+ * Whether a path is there.
+ */
+async function exists(path) {
+    try {
+        await lstat(path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Check that the verdicts agree with the assertions.
+ *
+ * The result is recomputed from the assertions and compared with what was
+ * recorded. A result that disagrees with its own evidence is not written,
+ * because a proof nobody can reconcile is not a proof.
+ */
+export function assertConsistent(result) {
+    if (!Array.isArray(result.scenarios) || result.scenarios.length === 0) {
+        throw new ProofError('the result carries no scenario', 'a result has to hold the scenarios that ran; a result with none was never run');
+    }
+    for (const scenario of result.scenarios) {
+        if (allPassed(scenario.assertions) !== scenario.passed) {
+            throw new ProofError('a scenario verdict disagrees with its assertions', 'the result was not written; the run and the assertions must be computed the same way');
+        }
+    }
+    if (result.scenarios.every((scenario) => scenario.passed) !== result.passed) {
+        throw new ProofError('the run verdict disagrees with the scenario verdicts', 'the result was not written; the run and the scenarios must be computed the same way');
+    }
+}
+/**
+ * The roots a working directory may sit in.
+ *
+ * The workspace, the OpenAgentsy worktrees of the same project, and whatever
+ * the caller adds.
+ */
+export function allowedRoots(options) {
+    const roots = [options.workspace, worktreeRoot(projectName(options.workspace))];
+    return [...roots, ...(options.allowedRoots ?? [])];
+}
+/**
+ * The real path of a root, or the root itself when it is not there.
+ */
+async function realPathOrSelf(path) {
+    try {
+        return await realpath(path);
+    }
+    catch {
+        return path;
+    }
+}
+/**
+ * The working directory of a scenario, checked.
+ *
+ * A relative one resolves against the base and has to land inside one of the
+ * roots. An absolute one has to land inside one of them as well, so a proof
+ * cannot ask for `/etc` or for a directory that only looks close.
+ *
+ * The check is made on the real paths, because a link inside the workspace
+ * can point out of it: a scenario that names a link would otherwise run
+ * somewhere the workspace does not cover.
+ */
+export async function resolveScenarioCwd(scenario, base, roots) {
+    const cwd = isAbsolute(scenario.cwd) ? resolve(scenario.cwd) : resolve(base, scenario.cwd);
+    if (!isInside(cwd, roots)) {
+        throw new ProofError(`the working directory of the scenario ${scenario.id} is outside the workspace`, 'point the scenario at a directory inside the workspace or an OpenAgentsy worktree');
+    }
+    let realCwd;
+    try {
+        realCwd = await realpath(cwd);
+    }
+    catch {
+        throw new ProofError(`the working directory of the scenario ${scenario.id} does not exist`, 'point the scenario at a directory that is there');
+    }
+    const realRoots = await Promise.all(roots.map(realPathOrSelf));
+    if (!isInside(realCwd, realRoots)) {
+        throw new ProofError(`the working directory of the scenario ${scenario.id} leaves the workspace through a symlink`, 'point the scenario at a real directory inside the workspace or an OpenAgentsy worktree');
+    }
+    return realCwd;
+}
+/**
+ * The artifact, read and checked.
+ *
+ * The schema is the contract: an artifact that does not match it is not a
+ * proof, and the issues say what is wrong with it.
+ */
+function parseArtifact(text) {
+    let parsedJson;
+    try {
+        parsedJson = JSON.parse(text);
+    }
+    catch (error) {
+        throw new ProofError('the eval artifact is not JSON', error instanceof Error ? error.message : 'the proof is malformed');
+    }
+    const parsed = EvalArtifact.safeParse(parsedJson);
+    if (!parsed.success) {
+        throw new ProofError('the eval artifact does not match the schema', parsed.error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`).join('; '));
+    }
+    return parsed.data;
+}
+/**
+ * The artifact, read and checked.
+ *
+ * The schema is the contract: an artifact that does not match it is not a
+ * proof, and the issues say what is wrong with it.
+ */
+export async function readArtifact(path) {
+    let text;
+    try {
+        text = await readFile(path, 'utf8');
+    }
+    catch (error) {
+        throw new ProofError('the eval artifact could not be read', error instanceof Error ? error.message : 'the proof is missing');
+    }
+    return parseArtifact(text);
+}
+/**
+ * Run an artifact and write the result.
+ *
+ * The result is written whatever the verdicts are: a failing run is the
+ * evidence that a lesson is not real, and it is worth keeping.
+ */
+export async function runEvalArtifact(options) {
+    const now = options.now ?? (() => new Date());
+    const startedAt = now();
+    let text;
+    try {
+        text = await readFile(options.artifactPath, 'utf8');
+    }
+    catch (error) {
+        throw new ProofError('the eval artifact could not be read', error instanceof Error ? error.message : 'the proof is missing');
+    }
+    const artifact = parseArtifact(text);
+    const lessonId = options.lessonId ?? artifact.lesson_id;
+    if (!isLessonId(lessonId)) {
+        throw new ProofError(`"${lessonId}" is not a lesson id`, 'a lesson id looks like oa-lesson-abcd1234');
+    }
+    const base = options.base ?? dirname(resolve(options.artifactPath));
+    const roots = allowedRoots(options);
+    const scenarios = [];
+    for (const scenario of artifact.scenarios) {
+        const cwd = await resolveScenarioCwd(scenario, base, roots);
+        const before = now();
+        const outcome = await runCommand(scenario.command, cwd, scenario.timeout_ms);
+        const after = now();
+        const assertions = assertionsOf(scenario, outcome);
+        scenarios.push({
+            id: scenario.id,
+            phase: scenario.phase,
+            command_sha256: commandHash(scenario.command),
+            exit_code: outcome.exitCode,
+            timed_out: outcome.timedOut,
+            duration_ms: Math.max(0, after.getTime() - before.getTime()),
+            assertions,
+            passed: allPassed(assertions),
+        });
+    }
+    const result = {
+        schema_version: '1',
+        lesson_id: lessonId,
+        artifact_sha256: hashOf(text),
+        started_at: startedAt.toISOString(),
+        finished_at: now().toISOString(),
+        passed: scenarios.every((scenario) => scenario.passed),
+        runs: 1,
+        scenarios,
+    };
+    assertConsistent(result);
+    const stateRoot = options.stateRoot ?? options.workspace;
+    const path = evalResultPath(stateRoot, lessonId, startedAt);
+    await writeEvalResult(path, result, stateRoot);
+    return result;
+}
+/**
+ * The git settings that keep a throwaway commit unsigned.
+ *
+ * The machine's own config may sign every commit, and the signing agent is
+ * not in the environment a child gets: the commit then fails, and the run
+ * fails with it (measured 2026-10-27: the 1Password agent was not reachable
+ * from the narrow environment, the commit was refused, and the worktree
+ * that followed it was made from a HEAD that did not exist). A throwaway
+ * commit buys nothing from a signature, so the runner turns the signing off
+ * on the git commands it spawns. The setting is on the command and not in
+ * the environment, because the environment stays narrow.
+ */
+const GIT_NO_SIGN = ['-c', 'commit.gpgsign=false'];
+/**
+ * The argv of a git command, with the runner's own settings in front of it.
+ *
+ * A git option only counts before the subcommand it belongs to, so the
+ * settings come first.
+ */
+export function gitCommand(args) {
+    return ['git', ...GIT_NO_SIGN, ...args];
+}
+/**
+ * Run a git command and wait for it.
+ *
+ * The worktree is made with git, so git is run with an argv and with the
+ * same narrow environment a proof gets. The command carries the runner's own
+ * settings, so a git that signs every commit does not decide whether the
+ * runner's work happens.
+ */
+async function git(cwd, args) {
+    const outcome = await runCommand(gitCommand(args), cwd, 60_000);
+    if (outcome.exitCode !== 0) {
+        throw new ProofError(`git ${args[0] ?? ''} failed`, outcome.stderr.trim() === '' ? 'the worktree could not be made' : outcome.stderr.trim());
+    }
+}
+/**
+ * Take the worktree and the directory it was made in back.
+ *
+ * A cleanup that fails is reported, because a directory left behind holds a
+ * whole checkout of the project.
+ */
+async function removeWorktree(repo, worktree, dir, added) {
+    const problems = [];
+    if (added) {
+        try {
+            await git(repo, ['worktree', 'remove', '--force', worktree]);
+        }
+        catch (error) {
+            problems.push(error instanceof Error ? error.message : String(error));
+        }
+    }
+    try {
+        await rm(dir, { recursive: true, force: true });
+    }
+    catch (error) {
+        problems.push(error instanceof Error ? error.message : String(error));
+    }
+    return problems.length === 0 ? undefined : problems.join('; ');
+}
+/**
+ * A temporary worktree, and what to do in it.
+ *
+ * The worktree is where the candidate change is applied: the baseline
+ * scenario has to fail on the current owner and pass with the change, and
+ * the change lives in a commit, so the commit is checked out apart from the
+ * tree that is being worked in.
+ *
+ * The directory is made before the worktree is added, so the cleanup has to
+ * run whether the add worked or not: a directory left behind holds a
+ * checkout of the project.
+ */
+export async function withTemporaryWorktree(options) {
+    const dir = await mkdtemp(join(tmpdir(), 'openagentsy-lesson-'));
+    const worktree = join(dir, 'worktree');
+    let added = false;
+    let cleaned = false;
+    const cleanup = async () => {
+        if (cleaned)
+            return undefined;
+        cleaned = true;
+        return removeWorktree(options.repo, worktree, dir, added);
+    };
+    try {
+        await git(options.repo, ['worktree', 'add', '--detach', worktree, options.ref]);
+        added = true;
+        const result = await options.run(worktree);
+        const problem = await cleanup();
+        if (problem !== undefined) {
+            throw new ProofError('the temporary worktree could not be removed', `the directory ${dir} is still there: ${problem}`);
+        }
+        return result;
+    }
+    catch (error) {
+        const problem = await cleanup();
+        const message = error instanceof Error ? error.message : String(error);
+        if (problem !== undefined) {
+            throw new ProofError(message, `the temporary worktree could not be removed either: ${problem}`);
+        }
+        throw error instanceof Error ? error : new Error(message);
+    }
+}
+/**
+ * Reproduce the baseline.
+ *
+ * The baseline scenario has to fail on the current owner and pass with the
+ * change applied in a temporary worktree. A baseline that passes on the
+ * current owner proves nothing: the behaviour was already there.
+ */
+export async function reproduceBaseline(options) {
+    const text = await readFile(options.artifactPath, 'utf8');
+    const artifact = parseArtifact(text);
+    const baselines = artifact.scenarios.filter((scenario) => scenario.phase === 'baseline');
+    if (baselines.length === 0) {
+        return {
+            baseline_failed: false,
+            with_change_passed: false,
+            passed: false,
+            reason: 'the artifact has no baseline scenario, so the baseline cannot be reproduced',
+        };
+    }
+    const before = await runEvalArtifact({
+        artifactPath: options.artifactPath,
+        workspace: options.workspace,
+        ...(options.lessonId === undefined ? {} : { lessonId: options.lessonId }),
+        ...(options.now === undefined ? {} : { now: options.now }),
+    });
+    const baselineFailed = before.scenarios
+        .filter((scenario) => scenario.phase === 'baseline')
+        .every((scenario) => !scenario.passed);
+    const withChange = await withTemporaryWorktree({
+        repo: options.repo,
+        ref: options.changeRef,
+        run: async (worktree) => runEvalArtifact({
+            artifactPath: options.artifactPath,
+            workspace: options.workspace,
+            base: worktree,
+            allowedRoots: [worktree],
+            ...(options.lessonId === undefined ? {} : { lessonId: options.lessonId }),
+            ...(options.now === undefined ? {} : { now: options.now }),
+        }),
+    });
+    const changePassed = withChange.scenarios
+        .filter((scenario) => scenario.phase === 'baseline')
+        .every((scenario) => scenario.passed);
+    return {
+        baseline_failed: baselineFailed,
+        with_change_passed: changePassed,
+        passed: baselineFailed && changePassed,
+        before,
+        after: withChange,
+    };
+}
+//# sourceMappingURL=runner.js.map

package/dist/eval/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAC9F,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACzE,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAClE,OAAO,EACL,YAAY,EAKZ,UAAU,GACX,MAAM,YAAY,CAAC;AAEpB;;;;;;;;;;;;;;;;;GAiBG;AAEH;;;;;;GAMG;AACH,MAAM,aAAa,GAAG;IACpB,MAAM;IACN,MAAM;IACN,QAAQ;IACR,MAAM;IACN,UAAU;IACV,QAAQ;IACR,UAAU;IACV,YAAY;IACZ,aAAa;IACb,YAAY;IACZ,SAAS;CACD,CAAC;AAEX;;;;;GAKG;AACH,MAAM,YAAY,GAAG,2BAA2B,CAAC;AAEjD,+DAA+D;AAC/D,MAAM,iBAAiB,GAAG,IAAI,GAAG,IAAI,CAAC;AAEtC,qEAAqE;AACrE,MAAM,UAAU,QAAQ,CAAC,MAAyB,OAAO,CAAC,GAAG;IAC3D,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,KAAK,KAAK,SAAS;YAAE,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IAC5C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,UAAU,CAAC,OAAgB;IACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,UAAU,CAClB,mCAAmC,EACnC,qFAAqF,CACtF,CAAC;IACJ,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClD,MAAM,IAAI,UAAU,CAClB,yDAAyD,EACzD,qFAAqF,CACtF,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,UAAU,CAClB,6BAA6B,EAC7B,mGAAmG,CACpG,CAAC;QACJ,CAAC;IACH,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAW,CAAC,EAAE,CAAC;QACpE,MAAM,IAAI,UAAU,CAClB,uCAAuC,EACvC,qFAAqF,CACtF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,MAAM,CAAC,IAAY;IACjC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACzD,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,WAAW,CAAC,OAA0B;IACpD,OAAO,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;AACzC,CAAC;AAUD;;;;;GAKG;AACH,SAAS,SAAS,CAAC,KAAmB;IACpC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;IACtB,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,OAAO;IACT,CAAC;IACD,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACxB,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,OAAiB,EAAE,GAAW,EAAE,SAAiB;IAC1E,UAAU,CAAC,OAAO,CAAC,CAAC;IACpB,OAAO,IAAI,OAAO,CAAC,CAAC,cAAc,EAAE,EAAE;QACpC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAW,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YAC1D,GAAG;YACH,GAAG,EAAE,QAAQ,EAAE;YACf,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,IAAI;YACd,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;SAClC,CAAC,CAAC;QACH,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,OAAO,GAAG,KAAK,CAAC;QAEpB,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,QAAQ,GAAG,IAAI,CAAC;YAChB,SAAS,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC,EAAE,SAAS,CAAC,CAAC;QAEd,MAAM,MAAM,GAAG,CAAC,OAAmB,EAAQ,EAAE;YAC3C,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,cAAc,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC,CAAC;QAEF,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YACzC,IAAI,MAAM,CAAC,MAAM,GAAG,iBAAiB;gBAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YACzC,IAAI,MAAM,CAAC,MAAM,GAAG,iBAAiB;gBAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE;YACjC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACpF,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAmB,EAAE,EAAE;YACxC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,QAAsB,EAAE,OAAmB;IACtE,MAAM,UAAU,GAAmB;QACjC,OAAO,EAAE,OAAO,CAAC,QAAQ,KAAK,IAAI,IAAI,OAAO,CAAC,QAAQ;KACvD,CAAC;IACF,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAC/B,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACnC,UAAU,CAAC,WAAW,CAAC,GAAG,OAAO,CAAC,QAAQ,KAAK,MAAM,CAAC,SAAS,CAAC;IAClE,CAAC;IACD,CAAC,MAAM,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;QACvD,UAAU,CAAC,mBAAmB,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC5E,CAAC,CAAC,CAAC;IACH,CAAC,MAAM,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;QACvD,UAAU,CAAC,mBAAmB,KAAK,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IACH,CAAC,MAAM,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;QACvD,UAAU,CAAC,mBAAmB,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC5E,CAAC,CAAC,CAAC;IACH,CAAC,MAAM,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;QACvD,UAAU,CAAC,mBAAmB,KAAK,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IACH,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,oCAAoC;AACpC,MAAM,UAAU,SAAS,CAAC,UAA0B;IAClD,OAAO,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC;AACpE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,SAAiB;IAC9C,OAAO,IAAI,CAAC,SAAS,EAAE,cAAc,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;AAC7D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAC,SAAiB,EAAE,QAAgB,EAAE,EAAQ;IAC1E,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,UAAU,CAClB,IAAI,QAAQ,sBAAsB,EAClC,2CAA2C,CAC5C,CAAC;IACJ,CAAC;IACD,MAAM,GAAG,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;IACtC,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACrD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,IAAI,KAAK,OAAO,CAAC,CAAC;IACpD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,UAAU,CAClB,kDAAkD,EAClD,6EAA6E,CAC9E,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,mBAAmB,CAAC,SAAiB,EAAE,GAAW;IAC/D,MAAM,IAAI,GAAG,QAAQ,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IACtC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,UAAU,CAClB,mDAAmD,EACnD,+CAA+C,CAChD,CAAC;IACJ,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC3C,IAAI,OAAO,GAAG,QAAQ,CAAC;IACvB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;QACxE,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACjC,IAAI,IAAuC,CAAC;QAC5C,IAAI,CAAC;YACH,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC;YAC1B,MAAM,IAAI,UAAU,CAClB,kBAAkB,OAAO,eAAe,EACxC,wDAAwD,CACzD,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,IAAY,EACZ,MAAkB,EAClB,SAAiB;IAEjB,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1B,MAAM,mBAAmB,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IAC1C,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEtC,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC3C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,CAAC;IACpC,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;QACzD,MAAM,IAAI,UAAU,CAClB,oDAAoD,EACpD,6DAA6D,CAC9D,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,OAAO,MAAM,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC;YAC1B,MAAM,IAAI,UAAU,CAClB,8BAA8B,EAC9B,wDAAwD,CACzD,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,CAAC;QACb,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,OAAO,OAAO,CAAC,CAAC;IAClE,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,IAAI,CACvB,MAAM,EACN,SAAS,CAAC,OAAO,GAAG,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,UAAU,EAChF,KAAK,CACN,CAAC;IACF,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACzE,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC3B,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,MAAM,CAAC,IAAY;IAChC,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAkB;IACjD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtE,MAAM,IAAI,UAAU,CAClB,gCAAgC,EAChC,+EAA+E,CAChF,CAAC;IACJ,CAAC;IACD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACxC,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;YACvD,MAAM,IAAI,UAAU,CAClB,kDAAkD,EAClD,sFAAsF,CACvF,CAAC;QACJ,CAAC;IACH,CAAC;IACD,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;QAC5E,MAAM,IAAI,UAAU,CAClB,sDAAsD,EACtD,qFAAqF,CACtF,CAAC;IACJ,CAAC;AACH,CAAC;AAyBD;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,OAAuB;IAClD,MAAM,KAAK,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,YAAY,CAAC,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAChF,OAAO,CAAC,GAAG,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAC;AACrD,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,cAAc,CAAC,IAAY;IACxC,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAAsB,EACtB,IAAY,EACZ,KAAe;IAEf,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC3F,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,UAAU,CAClB,yCAAyC,QAAQ,CAAC,EAAE,2BAA2B,EAC/E,mFAAmF,CACpF,CAAC;IACJ,CAAC;IACD,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,UAAU,CAClB,yCAAyC,QAAQ,CAAC,EAAE,iBAAiB,EACrE,iDAAiD,CAClD,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC;IAC/D,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,UAAU,CAClB,yCAAyC,QAAQ,CAAC,EAAE,yCAAyC,EAC7F,wFAAwF,CACzF,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;GAKG;AACH,SAAS,aAAa,CAAC,IAAY;IACjC,IAAI,UAAmB,CAAC;IACxB,IAAI,CAAC;QACH,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,UAAU,CAClB,+BAA+B,EAC/B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB,CAClE,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAClD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,UAAU,CAClB,6CAA6C,EAC7C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAC3F,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY;IAC7C,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACtC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,UAAU,CAClB,qCAAqC,EACrC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,sBAAsB,CAChE,CAAC;IACJ,CAAC;IACD,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAAuB;IAC3D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC;IAExB,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IACtD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,UAAU,CAClB,qCAAqC,EACrC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,sBAAsB,CAChE,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,QAAQ,CAAC,SAAS,CAAC;IACxD,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,UAAU,CAClB,IAAI,QAAQ,sBAAsB,EAClC,2CAA2C,CAC5C,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACpC,MAAM,SAAS,GAAyB,EAAE,CAAC;IAE3C,KAAK,MAAM,QAAQ,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;QAC1C,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC;QACrB,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC7E,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC;QACpB,MAAM,UAAU,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACnD,SAAS,CAAC,IAAI,CAAC;YACb,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,cAAc,EAAE,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC;YAC7C,SAAS,EAAE,OAAO,CAAC,QAAQ;YAC3B,SAAS,EAAE,OAAO,CAAC,QAAQ;YAC3B,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;YAC5D,UAAU;YACV,MAAM,EAAE,SAAS,CAAC,UAAU,CAAC;SAC9B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAe;QACzB,cAAc,EAAE,GAAG;QACnB,SAAS,EAAE,QAAQ;QACnB,eAAe,EAAE,MAAM,CAAC,IAAI,CAAC;QAC7B,UAAU,EAAE,SAAS,CAAC,WAAW,EAAE;QACnC,WAAW,EAAE,GAAG,EAAE,CAAC,WAAW,EAAE;QAChC,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;QACtD,IAAI,EAAE,CAAC;QACP,SAAS;KACV,CAAC;IAEF,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAEzB,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,SAAS,CAAC;IACzD,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC5D,MAAM,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAC/C,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,GAAG,CAAC,IAAI,EAAE,sBAAsB,CAAU,CAAC;AAE5D;;;;;GAKG;AACH,MAAM,UAAU,UAAU,CAAC,IAAuB;IAChD,OAAO,CAAC,KAAK,EAAE,GAAG,WAAW,EAAE,GAAG,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,GAAG,CAAC,GAAW,EAAE,IAAc;IAC5C,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;IAChE,IAAI,OAAO,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,UAAU,CAClB,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,SAAS,EAC7B,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,gCAAgC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CACxF,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,cAAc,CAC3B,IAAY,EACZ,QAAgB,EAChB,GAAW,EACX,KAAc;IAEd,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,KAAK,EAAE,CAAC;QACV,IAAI,CAAC;YACH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC/D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,QAAQ,CAAC,IAAI,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IACD,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAClD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CAAC,IAAI,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACxE,CAAC;IACD,OAAO,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACjE,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAI,OAI9C;IACC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,qBAAqB,CAAC,CAAC,CAAC;IACjE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IACvC,IAAI,KAAK,GAAG,KAAK,CAAC;IAClB,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,MAAM,OAAO,GAAG,KAAK,IAAiC,EAAE;QACtD,IAAI,OAAO;YAAE,OAAO,SAAS,CAAC;QAC9B,OAAO,GAAG,IAAI,CAAC;QACf,OAAO,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;IAC5D,CAAC,CAAC;IACF,IAAI,CAAC;QACH,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;QAChF,KAAK,GAAG,IAAI,CAAC;QACb,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC3C,MAAM,OAAO,GAAG,MAAM,OAAO,EAAE,CAAC;QAChC,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,MAAM,IAAI,UAAU,CAClB,6CAA6C,EAC7C,iBAAiB,GAAG,oBAAoB,OAAO,EAAE,CAClD,CAAC;QACJ,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,MAAM,OAAO,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,MAAM,IAAI,UAAU,CAClB,OAAO,EACP,uDAAuD,OAAO,EAAE,CACjE,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;IAC5D,CAAC;AACH,CAAC;AAkBD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,OAOvC;IACC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC;IACzF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO;YACL,eAAe,EAAE,KAAK;YACtB,kBAAkB,EAAE,KAAK;YACzB,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,6EAA6E;SACtF,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC;QACnC,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;QACzE,GAAG,CAAC,OAAO,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;KAC3D,CAAC,CAAC;IACH,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS;SACpC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,KAAK,UAAU,CAAC;SACnD,KAAK,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAEzC,MAAM,UAAU,GAAG,MAAM,qBAAqB,CAAC;QAC7C,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,GAAG,EAAE,OAAO,CAAC,SAAS;QACtB,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,CACtB,eAAe,CAAC;YACd,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,IAAI,EAAE,QAAQ;YACd,YAAY,EAAE,CAAC,QAAQ,CAAC;YACxB,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;YACzE,GAAG,CAAC,OAAO,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;SAC3D,CAAC;KACL,CAAC,CAAC;IACH,MAAM,YAAY,GAAG,UAAU,CAAC,SAAS;SACtC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,KAAK,UAAU,CAAC;SACnD,KAAK,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAExC,OAAO;QACL,eAAe,EAAE,cAAc;QAC/B,kBAAkB,EAAE,YAAY;QAChC,MAAM,EAAE,cAAc,IAAI,YAAY;QACtC,MAAM;QACN,KAAK,EAAE,UAAU;KAClB,CAAC;AACJ,CAAC"}

package/dist/eval/runner.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=runner.test.d.ts.map

package/dist/eval/runner.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.test.d.ts","sourceRoot":"","sources":["../../src/eval/runner.test.ts"],"names":[],"mappings":""}