@lesliechan721/agent-skill-eval 0.1.0-beta.7 → 0.1.0-beta.9

package/dist/codex-eval-app-server.js

@@ -0,0 +1,1366 @@
+import { spawn } from 'node:child_process';
+import { createHash, randomUUID } from 'node:crypto';
+import path from 'node:path';
+import fs from 'fs-extra';
+import { assertProcessTreeInspectionAvailable, CODEX_EVAL_PROCESS_TREE_ID_ENV, createChildProcessTreeMonitor, createDirectChildProcessMonitor, processTreeCleanupIsUnconfirmed, releaseChildProcessHandles, scrubProcessTreeId, } from './codex-eval-process-tree.js';
+import { assertCodexEvalCanStartWork, applyCodexEvalProviderEnvironment, createCodexEvalTemporaryRoot, materializeCodexEvalRuntimeHome, registerCodexEvalSignalCleanup, } from './codex-eval-runtime-config.js';
+import { codexEvalConfigOverrides, validateCodexEvalConfigIsolation, validateCodexEvalRpcIsolation, } from './codex-eval-isolation.js';
+import { validateAwSkillEvalConversationCaseDocument, } from './codex-eval-cases.js';
+const DEFAULT_OBSERVED_PROCESS_EVIDENCE = {
+    mode: 'observed',
+    inspectionAvailable: true,
+    userAcknowledged: false,
+    descendantCleanupGuaranteed: false,
+    limitation: 'Process-tree inspection observes and cleans known descendants but does not guarantee descendant cleanup.',
+};
+async function createEvalTemporaryRoot(prefix, workRoot) {
+    if (!workRoot)
+        return createCodexEvalTemporaryRoot(prefix);
+    await fs.ensureDir(workRoot);
+    return createCodexEvalTemporaryRoot(prefix, {
+        allocate: () => fs.mkdtemp(path.join(workRoot, prefix)),
+    });
+}
+export const CODEX_CONVERSATION_NETWORK_LIMITATION = 'Conversation app-server and tool execution network remains enabled because macOS Seatbelt cannot be nested reliably.';
+export const CODEX_CONVERSATION_FINAL_ANSWER_FALLBACK_LIMITATION = 'For final-answer fallback turns, expected_question_count is checked against answers_by_index length because structured questions are not observable.';
+export async function settleAppServerProcessCleanup(options) {
+    const diagnostics = await options.processTreeMonitor.terminate();
+    const cleanupUnconfirmed = processTreeCleanupIsUnconfirmed(diagnostics);
+    const closed = await options.waitForClose(1_000);
+    if (!closed)
+        releaseChildProcessHandles(options.child);
+    if (cleanupUnconfirmed)
+        throw new Error(diagnostics.join(' '));
+    if (closed)
+        return;
+    throw new Error(`Could not confirm Codex app-server process-tree termination. ${diagnostics.join(' ')}`.trim());
+}
+const PASSTHROUGH_ENV = new Set([
+    'PATH',
+    'LANG',
+    'LC_ALL',
+    'HTTP_PROXY',
+    'HTTPS_PROXY',
+    'ALL_PROXY',
+    'NO_PROXY',
+    'SSL_CERT_FILE',
+    'SSL_CERT_DIR',
+    'NODE_EXTRA_CA_CERTS',
+]);
+const CODEX_EVAL_APP_SERVER_OVERRIDES = [
+    'features.default_mode_request_user_input=true',
+    'shell_environment_policy.set.GIT_OPTIONAL_LOCKS="0"',
+];
+const DARWIN_SANDBOX_EXEC = '/usr/bin/sandbox-exec';
+class HostReadContainmentError extends Error {
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+export class CodexEvalAttemptControl {
+    options;
+    controller = new AbortController();
+    deadlineAt;
+    timer;
+    removeExternalAbort = () => undefined;
+    timedOut = false;
+    externallyAborted = false;
+    constructor(options) {
+        this.options = options;
+        if (options.timeoutMs !== undefined) {
+            this.deadlineAt = Date.now() + options.timeoutMs;
+            this.timer = setTimeout(() => {
+                this.timedOut = true;
+                this.controller.abort();
+            }, options.timeoutMs);
+            this.timer.unref();
+        }
+        const abort = () => {
+            this.externallyAborted = true;
+            this.controller.abort();
+        };
+        if (options.signal?.aborted)
+            abort();
+        else if (options.signal) {
+            options.signal.addEventListener('abort', abort, { once: true });
+            this.removeExternalAbort = () => options.signal?.removeEventListener('abort', abort);
+        }
+    }
+    get signal() {
+        return this.controller.signal;
+    }
+    get timeoutExpired() {
+        if (this.deadlineAt !== undefined && Date.now() >= this.deadlineAt) {
+            this.timedOut = true;
+            this.controller.abort();
+        }
+        return this.timedOut;
+    }
+    throwIfStopped(stage) {
+        if (this.deadlineAt !== undefined && Date.now() >= this.deadlineAt) {
+            this.timedOut = true;
+            this.controller.abort();
+        }
+        if (!this.controller.signal.aborted)
+            return;
+        if (this.timedOut) {
+            throw new Error(`${this.options.label} timed out after ${this.options.timeoutMs}ms during ${stage}.`);
+        }
+        if (this.externallyAborted)
+            throw new Error(`${this.options.label} aborted during ${stage}.`);
+        throw new Error(`${this.options.label} stopped during ${stage}.`);
+    }
+    remaining(stage) {
+        this.throwIfStopped(stage);
+        if (this.deadlineAt === undefined) {
+            throw new Error(`${this.options.label} has no deadline during ${stage}.`);
+        }
+        return Math.max(1, this.deadlineAt - Date.now());
+    }
+    rpcTimeout(stage, capMs = 30_000) {
+        this.throwIfStopped(stage);
+        if (this.deadlineAt === undefined)
+            return capMs;
+        return Math.max(1, Math.min(capMs, this.deadlineAt - Date.now()));
+    }
+    stopError(stage) {
+        try {
+            this.throwIfStopped(stage);
+        }
+        catch (error) {
+            return error;
+        }
+        return new Error(`${this.options.label} stopped during ${stage}.`);
+    }
+    dispose() {
+        if (this.timer)
+            clearTimeout(this.timer);
+        this.removeExternalAbort();
+    }
+}
+function comparablePath(value) {
+    const normalized = value.replaceAll('\\', '/');
+    return process.platform === 'win32' ? normalized.toLowerCase() : normalized;
+}
+function isPathInsideOrEqual(root, candidate) {
+    const relative = path.relative(root, candidate);
+    return relative === '' || (relative !== '..' &&
+        !relative.startsWith(`..${path.sep}`) &&
+        !path.isAbsolute(relative));
+}
+function containsForbiddenHostRoot(value, forbiddenHostRoots) {
+    if (typeof value === 'string') {
+        const candidate = comparablePath(value);
+        return forbiddenHostRoots.some((root) => candidate.includes(comparablePath(root)));
+    }
+    if (Array.isArray(value)) {
+        return value.some((entry) => containsForbiddenHostRoot(entry, forbiddenHostRoots));
+    }
+    if (isRecord(value)) {
+        return Object.values(value).some((entry) => containsForbiddenHostRoot(entry, forbiddenHostRoots));
+    }
+    return false;
+}
+function commandInputContainsForbiddenHostRoot(value, forbiddenHostRoots) {
+    const commandActions = Array.isArray(value.commandActions)
+        ? value.commandActions.filter(isRecord).map((action) => ({
+            command: action.command,
+            path: action.path,
+        }))
+        : [];
+    return containsForbiddenHostRoot([
+        value.command,
+        value.cwd,
+        commandActions,
+    ], forbiddenHostRoots);
+}
+function eventContainsForbiddenHostSourceInput(value, forbiddenHostRoots) {
+    if (!isRecord(value))
+        return false;
+    const params = isRecord(value.params) ? value.params : {};
+    if (value.method === 'item/commandExecution/requestApproval') {
+        return commandInputContainsForbiddenHostRoot(params, forbiddenHostRoots)
+            || containsForbiddenHostRoot(params.additionalPermissions, forbiddenHostRoots);
+    }
+    if (value.method === 'item/fileChange/requestApproval') {
+        return containsForbiddenHostRoot(params.grantRoot, forbiddenHostRoots);
+    }
+    const item = isRecord(params.item) ? params.item : null;
+    if (item?.type === 'commandExecution') {
+        return commandInputContainsForbiddenHostRoot(item, forbiddenHostRoots);
+    }
+    if (item?.type === 'fileChange' && Array.isArray(item.changes)) {
+        const paths = item.changes.filter(isRecord).map((change) => ({
+            path: change.path,
+            movePath: isRecord(change.kind) ? change.kind.move_path : undefined,
+        }));
+        return containsForbiddenHostRoot(paths, forbiddenHostRoots);
+    }
+    if (item?.type === 'imageView') {
+        return containsForbiddenHostRoot(item.path, forbiddenHostRoots);
+    }
+    return false;
+}
+function eventTraceContainsForbiddenHostSourceInput(eventTrace, forbiddenHostRoots) {
+    return eventTrace.some((event) => eventContainsForbiddenHostSourceInput(event, forbiddenHostRoots));
+}
+async function canonicalForbiddenHostRoots(roots) {
+    const values = await Promise.all(roots.map(async (root) => {
+        const resolved = path.resolve(root);
+        const real = await fs.realpath(resolved).catch(() => resolved);
+        return [resolved, real];
+    }));
+    return [...new Set(values.flat())];
+}
+function seatbeltPath(value) {
+    return JSON.stringify(value);
+}
+function hostReadContainmentProfile(options) {
+    return [
+        '(version 1)',
+        '(allow default)',
+        '(deny file-write*)',
+        `(allow file-write* (literal ${seatbeltPath('/dev/null')}))`,
+        ...options.writableRoots.map((root) => `(allow file-write* (subpath ${seatbeltPath(root)}))`),
+        ...options.unwritableRoots.map((root) => `(deny file-write* (subpath ${seatbeltPath(root)}))`),
+        ...options.forbiddenHostRoots.map((root) => `(deny file-read* (subpath ${seatbeltPath(root)}))`),
+    ].join('\n');
+}
+async function assertHostReadContainmentAvailable(profile) {
+    if (process.platform !== 'darwin') {
+        throw new HostReadContainmentError(`Codex conversation provider cannot start: OS-enforced host read containment is unavailable on ${process.platform}.`);
+    }
+    await new Promise((resolve, reject) => {
+        const child = spawn(DARWIN_SANDBOX_EXEC, ['-p', profile, '/usr/bin/true'], {
+            shell: false,
+            stdio: ['ignore', 'ignore', 'pipe'],
+        });
+        let stderr = '';
+        child.stderr?.on('data', (chunk) => {
+            stderr = `${stderr}${String(chunk)}`.slice(-4_000);
+        });
+        child.once('error', (error) => reject(new HostReadContainmentError(`Codex conversation provider cannot start OS-enforced host read containment: ${error.message}`)));
+        child.once('close', (code, signal) => {
+            if (code === 0)
+                resolve();
+            else
+                reject(new HostReadContainmentError('Codex conversation provider cannot start OS-enforced host read containment' +
+                    ` (code=${String(code)}, signal=${String(signal)}): ${stderr.trim() || 'no diagnostic'}`));
+        });
+    });
+}
+function requestId(value) {
+    const rawId = appServerRequestId(value);
+    return rawId === undefined ? undefined : String(rawId);
+}
+function appServerRequestId(value) {
+    return typeof value === 'string' || typeof value === 'number'
+        ? value
+        : undefined;
+}
+export function inspectAppServerConfigWarning(frame) {
+    if (frame.method !== 'configWarning')
+        return null;
+    const params = isRecord(frame.params) ? frame.params : {};
+    return typeof params.summary === 'string' ? params.summary : JSON.stringify(params);
+}
+function buildIsolatedEnv(sourceEnv, root, codexHome, runtime) {
+    const env = {};
+    for (const name of PASSTHROUGH_ENV) {
+        if (sourceEnv[name] !== undefined) {
+            env[name] = sourceEnv[name];
+        }
+    }
+    const home = path.join(root, 'home');
+    const temp = path.join(root, 'tmp');
+    return applyCodexEvalProviderEnvironment({
+        ...env,
+        HOME: home,
+        USERPROFILE: home,
+        CODEX_HOME: codexHome,
+        XDG_CACHE_HOME: path.join(root, 'xdg-cache'),
+        XDG_CONFIG_HOME: path.join(root, 'xdg-config'),
+        XDG_STATE_HOME: path.join(root, 'xdg-state'),
+        TMPDIR: temp,
+        TEMP: temp,
+        TMP: temp,
+        NO_COLOR: '1',
+        GIT_OPTIONAL_LOCKS: '0',
+    }, runtime);
+}
+async function snapshotWorkspace(root) {
+    const snapshot = new Map();
+    async function visit(current) {
+        const entries = await fs.readdir(current, { withFileTypes: true }).catch(() => []);
+        for (const entry of entries) {
+            const absolutePath = path.join(current, entry.name);
+            const relativePath = path.relative(root, absolutePath).split(path.sep).join('/');
+            if (entry.isDirectory()) {
+                snapshot.set(`${relativePath}/`, 'directory');
+                await visit(absolutePath);
+            }
+            else if (entry.isFile()) {
+                snapshot.set(relativePath, createHash('sha256').update(await fs.readFile(absolutePath)).digest('hex'));
+            }
+            else if (entry.isSymbolicLink()) {
+                snapshot.set(relativePath, `symlink:${await fs.readlink(absolutePath)}`);
+            }
+            else {
+                snapshot.set(relativePath, 'other');
+            }
+        }
+    }
+    await visit(root);
+    return snapshot;
+}
+function workspacePathMatchesPattern(relativePath, pattern) {
+    if (pattern.endsWith('/**')) {
+        const prefix = pattern.slice(0, -3);
+        return relativePath === prefix || relativePath.startsWith(`${prefix}/`);
+    }
+    return relativePath === pattern;
+}
+function workspaceChangeIsAllowed(relativePath, patterns) {
+    if (patterns.some((pattern) => workspacePathMatchesPattern(relativePath, pattern))) {
+        return true;
+    }
+    if (!relativePath.endsWith('/'))
+        return false;
+    const directoryPath = relativePath.slice(0, -1);
+    return patterns.some((pattern) => {
+        const allowedPath = pattern.endsWith('/**') ? pattern.slice(0, -3) : pattern;
+        return allowedPath.startsWith(`${directoryPath}/`);
+    });
+}
+function jsonContains(actual, expected) {
+    if (Array.isArray(expected)) {
+        return Array.isArray(actual)
+            && actual.length === expected.length
+            && expected.every((entry, index) => jsonContains(actual[index], entry));
+    }
+    if (isRecord(expected)) {
+        return isRecord(actual)
+            && Object.entries(expected).every(([key, value]) => Object.prototype.hasOwnProperty.call(actual, key) && jsonContains(actual[key], value));
+    }
+    return Object.is(actual, expected);
+}
+async function readWorkspaceArtifactEvidence(workspaceRoot, expectation) {
+    const absolutePath = path.join(workspaceRoot, ...expectation.path.split('/'));
+    const stat = await fs.lstat(absolutePath).catch((error) => {
+        if (error.code === 'ENOENT')
+            return null;
+        throw error;
+    });
+    if (!stat) {
+        return { path: expectation.path, exists: false, kind: 'absent' };
+    }
+    if (!stat.isFile() || stat.isSymbolicLink()) {
+        return { path: expectation.path, exists: true, kind: 'other' };
+    }
+    const content = await fs.readFile(absolutePath);
+    const evidence = {
+        path: expectation.path,
+        exists: true,
+        kind: 'file',
+        bytes: content.length,
+        sha256: createHash('sha256').update(content).digest('hex'),
+    };
+    if (content.length <= 64 * 1024 && !content.includes(0)) {
+        evidence.text = content.toString('utf8');
+    }
+    return evidence;
+}
+function validateWorkspaceArtifact(expectation, evidence) {
+    const diagnostics = [];
+    if (evidence.exists !== expectation.exists) {
+        diagnostics.push(`Artifact ${expectation.path} expected exists=${expectation.exists} but exists=${evidence.exists}.`);
+        return diagnostics;
+    }
+    if (!expectation.exists)
+        return diagnostics;
+    if (evidence.kind !== 'file') {
+        diagnostics.push(`Artifact ${expectation.path} must be a regular file.`);
+        return diagnostics;
+    }
+    const text = evidence.text;
+    if ((expectation.contains.length > 0 || expectation.notContains.length > 0 || expectation.json) && text === undefined) {
+        diagnostics.push(`Artifact ${expectation.path} is binary or exceeds the 64 KiB assertion limit.`);
+        return diagnostics;
+    }
+    for (const fragment of expectation.contains) {
+        if (!text.includes(fragment)) {
+            diagnostics.push(`Artifact ${expectation.path} is missing required text ${JSON.stringify(fragment)}.`);
+        }
+    }
+    for (const fragment of expectation.notContains) {
+        if (text.includes(fragment)) {
+            diagnostics.push(`Artifact ${expectation.path} contains forbidden text ${JSON.stringify(fragment)}.`);
+        }
+    }
+    if (expectation.json) {
+        let parsed;
+        try {
+            parsed = JSON.parse(text);
+        }
+        catch (error) {
+            diagnostics.push(`Artifact ${expectation.path} must contain valid JSON: ${error instanceof Error ? error.message : String(error)}.`);
+            return diagnostics;
+        }
+        if (!jsonContains(parsed, expectation.json)) {
+            diagnostics.push(`Artifact ${expectation.path} JSON does not contain the expected object subset.`);
+        }
+    }
+    return diagnostics;
+}
+async function isolateWorkspaceSkills(options) {
+    const skillsRoot = path.join(options.workspaceRoot, '.agents', 'skills');
+    const allowed = new Set(options.skillNames);
+    const entries = await fs.readdir(skillsRoot).catch((error) => {
+        if (error.code === 'ENOENT')
+            return [];
+        throw error;
+    });
+    await Promise.all(entries.map(async (entry) => {
+        if (allowed.has(entry))
+            return;
+        await fs.remove(path.join(skillsRoot, entry));
+    }));
+    const roots = options.skillNames.map((skillName) => path.join(skillsRoot, skillName));
+    for (const root of roots) {
+        if (!(await fs.pathExists(path.join(root, 'SKILL.md')))) {
+            throw new Error(`Materialized eval Skill is missing: ${root}`);
+        }
+    }
+    return roots;
+}
+function diffSnapshots(before, after) {
+    const paths = new Set([...before.keys(), ...after.keys()]);
+    return [...paths]
+        .filter((entry) => before.get(entry) !== after.get(entry))
+        .sort((left, right) => left.localeCompare(right));
+}
+export class EvalAppServerSession {
+    process;
+    processTreeId = randomUUID();
+    processTreeMonitor;
+    pending = new Map();
+    handlers = new Set();
+    failureHandlers = new Set();
+    stdoutBuffer = '';
+    stderrBuffer = '';
+    nextId = 1;
+    terminalError;
+    closing = false;
+    closed;
+    resolveClosed;
+    cleanupPromise;
+    unregisterSignalCleanup = () => undefined;
+    constructor(options) {
+        assertCodexEvalCanStartWork();
+        const processManagement = options.processManagement ?? DEFAULT_OBSERVED_PROCESS_EVIDENCE;
+        const managed = processManagement.mode === 'managed' || processManagement.mode === 'observed';
+        this.closed = new Promise((resolve) => {
+            this.resolveClosed = resolve;
+        });
+        const args = [
+            'app-server',
+            ...(options.isolation === 'strict' ? ['--strict-config'] : []),
+            '--listen',
+            'stdio://',
+            '--config',
+            'skills.include_instructions=true',
+            '--config',
+            `shell_environment_policy.exclude=${JSON.stringify([
+                'CODEX_HOME',
+                ...options.sensitiveEnvironmentNames,
+            ])}`,
+            '--config',
+            `model_provider=${JSON.stringify(options.modelProvider)}`,
+        ];
+        for (const override of codexEvalConfigOverrides(options.isolation)) {
+            args.push('--config', override);
+        }
+        for (const override of CODEX_EVAL_APP_SERVER_OVERRIDES) {
+            args.push('--config', override);
+        }
+        for (const override of options.configOverrides ?? []) {
+            args.push('--config', override);
+        }
+        const executable = options.hostReadContainmentProfile
+            ? DARWIN_SANDBOX_EXEC
+            : options.codexBin;
+        const executableArgs = options.hostReadContainmentProfile
+            ? ['-p', options.hostReadContainmentProfile, options.codexBin, ...args]
+            : args;
+        this.process = spawn(executable, executableArgs, {
+            cwd: options.cwd,
+            env: managed
+                ? { ...options.env, [CODEX_EVAL_PROCESS_TREE_ID_ENV]: this.processTreeId }
+                : { ...options.env },
+            shell: false,
+            detached: managed && process.platform !== 'win32',
+            stdio: ['pipe', 'pipe', 'pipe'],
+        });
+        this.processTreeMonitor = managed
+            ? createChildProcessTreeMonitor(this.process, { treeId: this.processTreeId })
+            : createDirectChildProcessMonitor(this.process);
+        this.process.stdout.setEncoding?.('utf8');
+        this.process.stderr.setEncoding?.('utf8');
+        this.process.stdout.on('data', (chunk) => {
+            this.handleChunk((typeof chunk === 'string' ? chunk : chunk.toString('utf8'))
+                .replaceAll(this.processTreeId, '[process-tree-id]'));
+        });
+        this.process.stderr.on('data', (chunk) => {
+            this.stderrBuffer = `${this.stderrBuffer}${String(chunk)}`
+                .replaceAll(this.processTreeId, '[process-tree-id]')
+                .slice(-32_000);
+        });
+        this.process.once('error', (error) => this.fail(error));
+        this.process.once('close', (code, signal) => {
+            this.resolveClosed?.();
+            if (!this.closing) {
+                this.fail(new Error(`codex app-server exited early (code=${code}, signal=${signal}).`));
+            }
+        });
+        this.unregisterSignalCleanup = registerCodexEvalSignalCleanup(() => this.cleanup());
+    }
+    onFrame(handler) {
+        this.handlers.add(handler);
+        return () => this.handlers.delete(handler);
+    }
+    onFailure(handler) {
+        this.failureHandlers.add(handler);
+        if (this.terminalError) {
+            handler(this.terminalError);
+        }
+        return () => this.failureHandlers.delete(handler);
+    }
+    async initialize(timeoutMs = 30_000) {
+        await this.sendRequest('initialize', {
+            clientInfo: {
+                name: 'aw-skill-eval-provider',
+                title: 'Agent Workflows Plugin Eval',
+                version: '1',
+            },
+            capabilities: {
+                experimentalApi: true,
+                requestAttestation: false,
+            },
+        }, timeoutMs);
+        await this.write({ method: 'initialized' });
+    }
+    async sendRequest(method, params, timeoutMs = 30_000) {
+        if (this.terminalError) {
+            throw this.terminalError;
+        }
+        const id = String(this.nextId++);
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                this.pending.delete(id);
+                reject(new Error(`Timed out waiting for app-server method ${method}.`));
+            }, timeoutMs);
+            this.pending.set(id, { method, resolve, reject, timer });
+            void this.write({ id, method, params }).catch((error) => {
+                clearTimeout(timer);
+                this.pending.delete(id);
+                reject(error instanceof Error ? error : new Error(String(error)));
+            });
+        });
+    }
+    async respond(id, result) {
+        await this.write({ id, result });
+    }
+    async cleanup() {
+        this.cleanupPromise ??= this.cleanupProcess();
+        return this.cleanupPromise;
+    }
+    async cleanupProcess() {
+        this.closing = true;
+        for (const pending of this.pending.values()) {
+            clearTimeout(pending.timer);
+            pending.reject(new Error('app-server session closed.'));
+        }
+        this.pending.clear();
+        try {
+            await settleAppServerProcessCleanup({
+                child: this.process,
+                processTreeMonitor: this.processTreeMonitor,
+                waitForClose: (timeoutMs) => this.waitForClose(timeoutMs),
+            });
+        }
+        finally {
+            this.unregisterSignalCleanup();
+        }
+    }
+    get stderr() {
+        return this.stderrBuffer.replaceAll(this.processTreeId, '[process-tree-id]');
+    }
+    scrub(value) {
+        return scrubProcessTreeId(value, this.processTreeId);
+    }
+    async waitForClose(timeoutMs) {
+        let timer;
+        const result = await Promise.race([
+            this.closed.then(() => true),
+            new Promise((resolve) => {
+                timer = setTimeout(() => resolve(false), timeoutMs);
+            }),
+        ]);
+        if (timer)
+            clearTimeout(timer);
+        return result;
+    }
+    async write(frame) {
+        if (!this.process.stdin) {
+            throw new Error('codex app-server stdin is unavailable.');
+        }
+        await new Promise((resolve, reject) => {
+            this.process.stdin.write(`${JSON.stringify(frame)}\n`, (error) => {
+                if (error) {
+                    reject(error);
+                }
+                else {
+                    resolve();
+                }
+            });
+        });
+    }
+    handleChunk(chunk) {
+        this.stdoutBuffer += chunk;
+        let newlineIndex = this.stdoutBuffer.indexOf('\n');
+        while (newlineIndex >= 0) {
+            const line = this.stdoutBuffer.slice(0, newlineIndex).replace(/\r$/u, '');
+            this.stdoutBuffer = this.stdoutBuffer.slice(newlineIndex + 1);
+            this.handleLine(line);
+            newlineIndex = this.stdoutBuffer.indexOf('\n');
+        }
+    }
+    handleLine(line) {
+        if (!line.trim()) {
+            return;
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(line);
+        }
+        catch {
+            this.fail(new Error(`Invalid app-server JSONL: ${line.slice(0, 240)}`));
+            return;
+        }
+        if (!isRecord(parsed)) {
+            return;
+        }
+        const id = requestId(parsed.id);
+        if (id && ('result' in parsed || 'error' in parsed)) {
+            const pending = this.pending.get(id);
+            if (!pending) {
+                return;
+            }
+            clearTimeout(pending.timer);
+            this.pending.delete(id);
+            if ('error' in parsed) {
+                pending.reject(new Error(`app-server ${pending.method} failed: ${JSON.stringify(parsed.error)}`));
+            }
+            else {
+                pending.resolve(parsed.result);
+            }
+            return;
+        }
+        for (const handler of this.handlers) {
+            void Promise.resolve(handler(parsed)).catch((error) => this.fail(error instanceof Error ? error : new Error(String(error))));
+        }
+    }
+    fail(error) {
+        if (this.terminalError) {
+            return;
+        }
+        this.terminalError = error;
+        for (const pending of this.pending.values()) {
+            clearTimeout(pending.timer);
+            pending.reject(error);
+        }
+        this.pending.clear();
+        for (const handler of this.failureHandlers) {
+            handler(error);
+        }
+    }
+}
+async function collectStrictIsolationEvidence(options) {
+    const [config, requirements] = await Promise.all([
+        options.session.sendRequest('config/read', {
+            includeLayers: true,
+            cwd: options.workspaceRoot,
+        }, options.control?.rpcTimeout('config/read') ?? 30_000),
+        options.session.sendRequest('configRequirements/read', {}, options.control?.rpcTimeout('configRequirements/read') ?? 30_000),
+    ]);
+    validateCodexEvalConfigIsolation({
+        config,
+        requirements,
+        codexHome: options.codexHome,
+        workspaceRoot: options.workspaceRoot,
+    });
+    const [skills, mcpServers] = await Promise.all([
+        options.session.sendRequest('skills/list', {
+            cwds: [options.workspaceRoot],
+            forceReload: true,
+        }, options.control?.rpcTimeout('skills/list') ?? 30_000),
+        options.session.sendRequest('mcpServerStatus/list', {
+            cursor: null,
+            limit: 100,
+            detail: 'toolsAndAuthOnly',
+            threadId: null,
+        }, options.control?.rpcTimeout('mcpServerStatus/list') ?? 30_000),
+    ]);
+    return validateCodexEvalRpcIsolation({
+        config,
+        requirements,
+        skills,
+        mcpServers,
+        codexHome: options.codexHome,
+        workspaceRoot: options.workspaceRoot,
+        allowedSkillRoots: options.allowedSkillRoots,
+    });
+}
+function normalizeScriptedReply(value, label) {
+    if (typeof value === 'string') {
+        return {
+            expectedQuestionCount: 1,
+            answersByIndex: [value],
+            unexpectedQuestionAnswer: '我暂时不知道，请先处理已经回答的部分。',
+        };
+    }
+    if (!isRecord(value)) {
+        throw new Error(`${label} must be a string or object.`);
+    }
+    if (!Array.isArray(value.answers_by_index) || value.answers_by_index.some((entry) => typeof entry !== 'string')) {
+        throw new Error(`${label}.answers_by_index must be an array of strings.`);
+    }
+    const expectedQuestionCount = value.expected_question_count;
+    if (expectedQuestionCount !== undefined &&
+        (!Number.isInteger(expectedQuestionCount) || expectedQuestionCount < 0)) {
+        throw new Error(`${label}.expected_question_count must be a non-negative integer.`);
+    }
+    const fallback = value.unexpected_question_answer;
+    if (typeof fallback !== 'string' || fallback.trim() === '') {
+        throw new Error(`${label}.unexpected_question_answer must be a non-empty string.`);
+    }
+    return {
+        ...(expectedQuestionCount === undefined
+            ? {}
+            : { expectedQuestionCount: expectedQuestionCount }),
+        answersByIndex: value.answers_by_index,
+        unexpectedQuestionAnswer: fallback,
+    };
+}
+function normalizeConversationCase(value, index) {
+    const review = isRecord(value.review) ? value.review : {};
+    if (typeof value.id !== 'string' ||
+        typeof value.prompt !== 'string' ||
+        !Array.isArray(value.scripted_replies) ||
+        typeof value.expected_output !== 'string' ||
+        !Array.isArray(review.checks) ||
+        review.checks.some((entry) => typeof entry !== 'string')) {
+        throw new Error(`evals[${index}] has an invalid behavior case shape.`);
+    }
+    return {
+        id: value.id.trim(),
+        prompt: value.prompt,
+        scriptedReplies: value.scripted_replies.map((entry, replyIndex) => normalizeScriptedReply(entry, `evals[${index}].scripted_replies[${replyIndex}]`)),
+        expectedOutput: value.expected_output,
+        checks: review.checks,
+        workspaceFiles: isRecord(value.workspace_files)
+            ? Object.fromEntries(Object.entries(value.workspace_files).map(([relativePath, content]) => [
+                relativePath,
+                content,
+            ]))
+            : {},
+        workspaceUnchanged: review.workspace_unchanged === true,
+        sandbox: value.sandbox === 'workspace-write' ? 'workspace-write' : 'read-only',
+        allowedWorkspaceChanges: Array.isArray(review.allowed_workspace_changes)
+            ? review.allowed_workspace_changes
+            : [],
+        artifacts: Array.isArray(review.artifacts)
+            ? review.artifacts.map((artifact) => {
+                const row = artifact;
+                return {
+                    path: row.path,
+                    exists: row.exists !== false,
+                    contains: Array.isArray(row.contains) ? row.contains : [],
+                    notContains: Array.isArray(row.not_contains) ? row.not_contains : [],
+                    ...(isRecord(row.json) ? { json: row.json } : {}),
+                };
+            })
+            : [],
+        environmentPathPrepend: isRecord(value.environment) &&
+            Array.isArray(value.environment.pathPrepend)
+            ? value.environment.pathPrepend
+            : [],
+    };
+}
+export function normalizeConversationProviderCase(options) {
+    const [value] = validateAwSkillEvalConversationCaseDocument({ skill_name: options.expectedSkill, evals: [options.value] }, options.expectedSkill, 'conversation provider case', options.fixtureRoot, options.suitePathPrepend);
+    return normalizeConversationCase(value, 0);
+}
+async function assertFixtureTreeSafe(root, target, label) {
+    const rootRealPath = await fs.realpath(root);
+    const targetRealPath = await fs.realpath(target);
+    if (!isPathInsideOrEqual(rootRealPath, targetRealPath)) {
+        throw new Error(`${label} must stay inside its Eval fixture root.`);
+    }
+    const relative = path.relative(root, target);
+    let current = root;
+    for (const segment of relative.split(path.sep).filter(Boolean)) {
+        current = path.join(current, segment);
+        const stat = await fs.lstat(current);
+        if (stat.isSymbolicLink())
+            throw new Error(`${label} must not contain symbolic links.`);
+    }
+    const stat = await fs.lstat(target);
+    if (stat.isFile())
+        return;
+    if (!stat.isDirectory())
+        throw new Error(`${label} must be a regular file or directory.`);
+    const entries = await fs.readdir(target);
+    for (const entry of entries) {
+        await assertFixtureTreeSafe(root, path.join(target, entry), label);
+    }
+}
+async function makeFixtureExecutable(target) {
+    const stat = await fs.lstat(target);
+    if (stat.isFile()) {
+        await fs.chmod(target, 0o755);
+        return;
+    }
+    if (!stat.isDirectory())
+        return;
+    for (const entry of await fs.readdir(target)) {
+        await makeFixtureExecutable(path.join(target, entry));
+    }
+}
+async function materializeConversationEnvironment(options) {
+    const pathEntries = [];
+    const materialized = new Map();
+    for (const { fixtureRoot, relativePath } of options.fixtures) {
+        const source = path.resolve(fixtureRoot, relativePath);
+        await assertFixtureTreeSafe(fixtureRoot, source, `Conversation PATH fixture ${JSON.stringify(relativePath)}`);
+        const target = path.resolve(options.workspaceRoot, relativePath);
+        if (!isPathInsideOrEqual(options.workspaceRoot, target)) {
+            throw new Error(`Conversation PATH fixture must stay inside the eval workspace: ${relativePath}`);
+        }
+        const sourceRealPath = await fs.realpath(source);
+        const previousSource = materialized.get(target);
+        if (previousSource) {
+            if (previousSource !== sourceRealPath) {
+                throw new Error(`Conversation PATH fixtures conflict at: ${relativePath}`);
+            }
+            continue;
+        }
+        if (await fs.pathExists(target)) {
+            throw new Error(`Conversation PATH fixture conflicts with materialized subject: ${relativePath}`);
+        }
+        await fs.copy(source, target, { dereference: false, errorOnExist: true, overwrite: false });
+        await makeFixtureExecutable(target);
+        materialized.set(target, sourceRealPath);
+        const stat = await fs.lstat(target);
+        pathEntries.push(stat.isDirectory() ? target : path.dirname(target));
+    }
+    return pathEntries;
+}
+async function materializeConversationWorkspaceFiles(workspaceRoot, workspaceFiles) {
+    for (const [relativePath, content] of Object.entries(workspaceFiles)) {
+        const segments = relativePath.split('/');
+        const fileName = segments.pop();
+        let parent = workspaceRoot;
+        for (const segment of segments) {
+            parent = path.join(parent, segment);
+            if (await fs.pathExists(parent)) {
+                const stat = await fs.lstat(parent);
+                if (!stat.isDirectory() || stat.isSymbolicLink()) {
+                    throw new Error(`Conversation workspace file has an unsafe parent path: ${relativePath}`);
+                }
+            }
+            else {
+                await fs.mkdir(parent);
+            }
+        }
+        const target = path.join(parent, fileName);
+        if (await fs.pathExists(target)) {
+            throw new Error(`Conversation workspace file conflicts with materialized subject: ${relativePath}`);
+        }
+        await fs.writeFile(target, content, 'utf8');
+    }
+}
+export function extractAppServerThreadId(result) {
+    if (!isRecord(result) || !isRecord(result.thread) || typeof result.thread.id !== 'string') {
+        throw new Error('app-server thread/start response is missing thread.id.');
+    }
+    return result.thread.id;
+}
+function finalAnswerReplyText(scripted) {
+    return [
+        'Scripted user reply (answers_by_index order):',
+        ...scripted.answersByIndex.map((answer, index) => `${index + 1}. ${answer}`),
+    ].join('\n');
+}
+async function runConversationOnce(options) {
+    assertCodexEvalCanStartWork();
+    const control = new CodexEvalAttemptControl({
+        label: 'conversation evaluation',
+        ...(options.manageTimeout === false ? {} : { timeoutMs: options.timeoutMs }),
+        ...(options.signal ? { signal: options.signal } : {}),
+    });
+    const temporaryRoot = await createEvalTemporaryRoot('conversation-', options.forbiddenHostRoots.length > 0 ? undefined : options.workRoot);
+    const runRoot = await fs.realpath(temporaryRoot.root);
+    const workspaceRoot = path.join(runRoot, 'project');
+    const codexHome = path.join(runRoot, 'codex-home');
+    const diagnostics = [];
+    const scriptedReplyMismatches = [];
+    const questions = [];
+    const fallbackUserReplies = [];
+    const finalMessages = [];
+    const eventTrace = [];
+    let usage = null;
+    let isolationEvidence = null;
+    let session;
+    let removeHandler;
+    let removeFailureHandler;
+    let removeAbortHandler;
+    const startedAt = Date.now();
+    try {
+        options.onStage?.('workspace-materialization-started');
+        const containmentProfile = hostReadContainmentProfile({
+            forbiddenHostRoots: options.forbiddenHostRoots,
+            writableRoots: [runRoot],
+            unwritableRoots: options.evalCase.sandbox === 'read-only' ? [workspaceRoot] : [],
+        });
+        options.onStage?.('containment-admission-started');
+        await assertHostReadContainmentAvailable(containmentProfile);
+        options.onStage?.('containment-admitted');
+        await fs.copy(options.sourceProjectRoot, workspaceRoot, { dereference: false });
+        const environmentPathEntries = await materializeConversationEnvironment({
+            workspaceRoot,
+            fixtures: options.environmentFixtures,
+        });
+        await materializeConversationWorkspaceFiles(workspaceRoot, options.evalCase.workspaceFiles);
+        let allowedSkillRoots = [];
+        if (options.isolation === 'strict') {
+            const visibleSkills = options.variant === 'with_skill'
+                ? options.skillNames
+                : options.skillNames.filter((skillName) => skillName !== options.skillName);
+            allowedSkillRoots = await isolateWorkspaceSkills({
+                workspaceRoot,
+                skillNames: visibleSkills,
+            });
+        }
+        else if (options.variant === 'without_skill') {
+            await fs.remove(path.join(workspaceRoot, '.agents', 'skills', options.skillName));
+        }
+        const env = buildIsolatedEnv(options.sourceEnv, runRoot, codexHome, options.codexRuntime);
+        env.PATH = [...environmentPathEntries, env.PATH].filter(Boolean).join(path.delimiter);
+        await Promise.all([
+            fs.ensureDir(env.HOME),
+            fs.ensureDir(env.TMPDIR),
+            materializeCodexEvalRuntimeHome(options.codexRuntime, codexHome, {
+                trustedProject: workspaceRoot,
+            }),
+        ]);
+        options.onStage?.('workspace-materialized');
+        const before = await snapshotWorkspace(workspaceRoot);
+        if (options.processManagement.mode === 'managed' ||
+            options.processManagement.mode === 'observed') {
+            await assertProcessTreeInspectionAvailable();
+        }
+        control.throwIfStopped('initialize');
+        assertCodexEvalCanStartWork();
+        session = new EvalAppServerSession({
+            codexBin: options.codexBin,
+            cwd: workspaceRoot,
+            env,
+            modelProvider: options.codexRuntime.modelProvider,
+            sensitiveEnvironmentNames: options.codexRuntime.sensitiveEnvironmentNames,
+            isolation: options.isolation,
+            processManagement: options.processManagement,
+            hostReadContainmentProfile: containmentProfile,
+        });
+        options.onStage?.('session-started');
+        let replyStep = 0;
+        let turnResolve;
+        let turnReject;
+        let turnUsedRequestUserInput = false;
+        let turnProducedFinalAnswer = false;
+        const abort = () => {
+            const error = control.stopError('conversation turn wait');
+            turnReject?.(error);
+            void session?.cleanup().catch((cleanupError) => {
+                turnReject?.(cleanupError instanceof Error ? cleanupError : new Error(String(cleanupError)));
+            });
+        };
+        if (control.signal.aborted)
+            abort();
+        else {
+            control.signal.addEventListener('abort', abort, { once: true });
+            removeAbortHandler = () => control.signal.removeEventListener('abort', abort);
+        }
+        removeFailureHandler = session.onFailure((error) => turnReject?.(error));
+        removeHandler = session.onFrame(async (frame) => {
+            const method = typeof frame.method === 'string' ? frame.method : '';
+            const params = isRecord(frame.params) ? frame.params : {};
+            if (!method.startsWith('remoteControl/') && !method.startsWith('account/')) {
+                eventTrace.push(frame);
+            }
+            const configWarning = inspectAppServerConfigWarning(frame);
+            if (options.isolation === 'strict' && configWarning) {
+                throw new Error(`Strict Codex eval received a config warning: ${configWarning}`);
+            }
+            if (method === 'item/tool/requestUserInput') {
+                turnUsedRequestUserInput = true;
+                const responseId = appServerRequestId(frame.id);
+                const id = requestId(responseId);
+                if (!id || responseId === undefined || !Array.isArray(params.questions)) {
+                    throw new Error('requestUserInput frame is missing id or questions.');
+                }
+                const scripted = options.evalCase.scriptedReplies[replyStep];
+                if (!scripted) {
+                    scriptedReplyMismatches.push(`Unexpected request_user_input step ${replyStep + 1}.`);
+                }
+                const fallback = scripted?.unexpectedQuestionAnswer ?? '请停止提问并交还控制。';
+                const normalizedQuestions = params.questions.map((value, index) => {
+                    const row = isRecord(value) ? value : {};
+                    return {
+                        id: typeof row.id === 'string' ? row.id : `question-${index}`,
+                        header: typeof row.header === 'string' ? row.header : '',
+                        question: typeof row.question === 'string' ? row.question : '',
+                        options: Array.isArray(row.options)
+                            ? row.options.filter(isRecord).map((option) => ({
+                                label: typeof option.label === 'string' ? option.label : '',
+                                description: typeof option.description === 'string' ? option.description : '',
+                            }))
+                            : null,
+                    };
+                });
+                if (scripted?.expectedQuestionCount !== undefined &&
+                    scripted.expectedQuestionCount !== normalizedQuestions.length) {
+                    scriptedReplyMismatches.push(`request_user_input step ${replyStep + 1} expected ${scripted.expectedQuestionCount} questions but received ${normalizedQuestions.length}.`);
+                }
+                const answers = Object.fromEntries(normalizedQuestions.map((question, index) => [
+                    question.id,
+                    [scripted?.answersByIndex[index] ?? fallback],
+                ]));
+                questions.push({
+                    requestId: id,
+                    step: replyStep,
+                    questions: normalizedQuestions,
+                    answers,
+                });
+                replyStep += 1;
+                await session.respond(responseId, {
+                    answers: Object.fromEntries(Object.entries(answers).map(([key, value]) => [key, { answers: value }])),
+                });
+                return;
+            }
+            if (method === 'item/completed' && isRecord(params.item)) {
+                const item = params.item;
+                if (item.type === 'agentMessage' &&
+                    item.phase === 'final_answer' &&
+                    typeof item.text === 'string') {
+                    turnProducedFinalAnswer = true;
+                    finalMessages.push(item.text);
+                }
+            }
+            if (method === 'thread/tokenUsage/updated') {
+                usage = params.tokenUsage ?? usage;
+            }
+            if (method === 'turn/completed') {
+                const turn = isRecord(params.turn) ? params.turn : {};
+                turnResolve?.({ complete: turn.status === 'completed' });
+            }
+            if (method === 'error') {
+                turnReject?.(new Error(`app-server error: ${JSON.stringify(params)}`));
+            }
+        });
+        await session.initialize(control.rpcTimeout('initialize'));
+        control.throwIfStopped('initialize');
+        options.onStage?.('session-initialized');
+        if (options.isolation === 'strict') {
+            isolationEvidence = await collectStrictIsolationEvidence({
+                session,
+                workspaceRoot,
+                codexHome,
+                allowedSkillRoots,
+                control,
+            });
+        }
+        control.throwIfStopped('thread/start');
+        const threadResult = await session.sendRequest('thread/start', {
+            cwd: workspaceRoot,
+            approvalPolicy: 'never',
+            sandbox: options.evalCase.sandbox,
+            ephemeral: false,
+            ...(options.model ? { model: options.model } : {}),
+        }, control.rpcTimeout('thread/start'));
+        const threadId = extractAppServerThreadId(threadResult);
+        options.onStage?.('thread-started');
+        let nextTurnText = options.evalCase.prompt;
+        let completed = { complete: false };
+        while (true) {
+            turnUsedRequestUserInput = false;
+            turnProducedFinalAnswer = false;
+            const turnDone = new Promise((resolve, reject) => {
+                turnResolve = resolve;
+                turnReject = reject;
+            });
+            void turnDone.catch(() => undefined);
+            control.throwIfStopped('turn/start');
+            await session.sendRequest('turn/start', {
+                threadId,
+                input: [{ type: 'text', text: nextTurnText, text_elements: [] }],
+                cwd: workspaceRoot,
+                approvalPolicy: 'never',
+                sandboxPolicy: {
+                    type: 'externalSandbox',
+                    networkAccess: 'enabled',
+                },
+                ...(options.model ? { model: options.model } : {}),
+            }, control.rpcTimeout('turn/start'));
+            options.onStage?.('turn-started');
+            completed = await turnDone;
+            control.throwIfStopped('turn wait');
+            options.onStage?.('turn-completed');
+            turnResolve = undefined;
+            turnReject = undefined;
+            const scripted = options.evalCase.scriptedReplies[replyStep];
+            if (!completed.complete ||
+                !scripted ||
+                turnUsedRequestUserInput ||
+                !turnProducedFinalAnswer) {
+                break;
+            }
+            if (scripted.expectedQuestionCount !== undefined &&
+                scripted.expectedQuestionCount !== scripted.answersByIndex.length) {
+                scriptedReplyMismatches.push(`Final-answer fallback step ${replyStep + 1} cannot observe structured questions: expected_question_count=${scripted.expectedQuestionCount}, but answers_by_index contains ${scripted.answersByIndex.length} answer(s).`);
+            }
+            fallbackUserReplies.push({
+                step: replyStep,
+                eventTraceIndex: eventTrace.length,
+                answers: [...scripted.answersByIndex],
+            });
+            replyStep += 1;
+            nextTurnText = finalAnswerReplyText(scripted);
+        }
+        removeHandler();
+        removeHandler = undefined;
+        removeFailureHandler();
+        removeFailureHandler = undefined;
+        await session.cleanup();
+        control.throwIfStopped('cleanup');
+        const after = await snapshotWorkspace(workspaceRoot);
+        const workspaceChanges = diffSnapshots(before, after);
+        const workspaceArtifacts = await Promise.all(options.evalCase.artifacts.map((expectation) => readWorkspaceArtifactEvidence(workspaceRoot, expectation)));
+        const hostSourceAccessAbsent = options.forbiddenHostRoots.length > 0
+            ? !eventTraceContainsForbiddenHostSourceInput(eventTrace, options.forbiddenHostRoots)
+            : null;
+        if (hostSourceAccessAbsent === false) {
+            diagnostics.push('Codex eval accessed a forbidden host source root.');
+        }
+        if (replyStep < options.evalCase.scriptedReplies.length) {
+            scriptedReplyMismatches.push(`${options.evalCase.scriptedReplies.length - replyStep} scripted reply step(s) were unused.`);
+        }
+        if (options.evalCase.workspaceUnchanged && workspaceChanges.length > 0) {
+            diagnostics.push(`Workspace changed: ${workspaceChanges.join(', ')}`);
+        }
+        if (options.evalCase.sandbox === 'workspace-write') {
+            const unexpectedChanges = workspaceChanges.filter((relativePath) => !workspaceChangeIsAllowed(relativePath, options.evalCase.allowedWorkspaceChanges));
+            if (unexpectedChanges.length > 0) {
+                diagnostics.push(`Workspace changed outside the allowed paths: ${unexpectedChanges.join(', ')}`);
+            }
+        }
+        if (options.variant === 'with_skill') {
+            diagnostics.push(...scriptedReplyMismatches);
+            for (const [index, expectation] of options.evalCase.artifacts.entries()) {
+                diagnostics.push(...validateWorkspaceArtifact(expectation, workspaceArtifacts[index]));
+            }
+        }
+        return session.scrub({
+            variant: options.variant,
+            complete: completed.complete,
+            mechanicalPassed: completed.complete && diagnostics.length === 0,
+            hostSourceAccessAbsent,
+            scriptedReplyMismatches,
+            durationMs: Date.now() - startedAt,
+            questions,
+            fallbackUserReplies,
+            finalMessages,
+            usage,
+            workspaceChanges,
+            workspaceArtifacts,
+            diagnostics,
+            eventTrace,
+            isolation: isolationEvidence,
+            limitations: [
+                CODEX_CONVERSATION_NETWORK_LIMITATION,
+                CODEX_CONVERSATION_FINAL_ANSWER_FALLBACK_LIMITATION,
+            ],
+            ...(session.stderr.trim() ? { stderr: session.stderr.trim() } : {}),
+        });
+    }
+    catch (error) {
+        if (error instanceof HostReadContainmentError)
+            throw error;
+        const hostSourceAccessAbsent = options.forbiddenHostRoots.length > 0 &&
+            eventTraceContainsForbiddenHostSourceInput(eventTrace, options.forbiddenHostRoots)
+            ? false
+            : null;
+        const evidence = {
+            variant: options.variant,
+            complete: false,
+            mechanicalPassed: false,
+            hostSourceAccessAbsent,
+            scriptedReplyMismatches,
+            durationMs: Date.now() - startedAt,
+            questions,
+            fallbackUserReplies,
+            finalMessages,
+            usage,
+            workspaceChanges: [],
+            workspaceArtifacts: [],
+            diagnostics: [error instanceof Error ? error.stack ?? error.message : String(error)],
+            eventTrace,
+            isolation: isolationEvidence,
+            limitations: [
+                CODEX_CONVERSATION_NETWORK_LIMITATION,
+                CODEX_CONVERSATION_FINAL_ANSWER_FALLBACK_LIMITATION,
+            ],
+            ...(session?.stderr.trim() ? { stderr: session.stderr.trim() } : {}),
+        };
+        return session ? session.scrub(evidence) : evidence;
+    }
+    finally {
+        removeHandler?.();
+        removeFailureHandler?.();
+        removeAbortHandler?.();
+        try {
+            await session?.cleanup();
+        }
+        catch (error) {
+            throw new Error(`Conversation eval app-server cleanup failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        finally {
+            try {
+                await temporaryRoot.cleanup();
+            }
+            finally {
+                control.dispose();
+            }
+        }
+    }
+}
+export async function runCodexConversationProviderCase(options) {
+    return runConversationOnce({
+        sourceProjectRoot: options.sourceProjectRoot,
+        evalCase: options.evalCase,
+        variant: options.skillMode === 'without-skill' ? 'without_skill' : 'with_skill',
+        skillName: options.skillName,
+        skillNames: options.skillNames,
+        codexBin: options.codexBin,
+        ...(options.model ? { model: options.model } : {}),
+        codexRuntime: options.codexRuntime,
+        timeoutMs: options.timeoutMs,
+        sourceEnv: options.sourceEnv ?? process.env,
+        isolation: options.isolation,
+        processManagement: options.processManagement ?? {
+            mode: 'unmanaged-confirmed',
+            inspectionAvailable: false,
+            userAcknowledged: true,
+            descendantCleanupGuaranteed: false,
+            limitation: 'The AwSkillEval engine owns the provider process tree and timeout.',
+        },
+        forbiddenHostRoots: await canonicalForbiddenHostRoots([options.sourceProjectRoot, ...(options.forbiddenHostRoots ?? [])]),
+        environmentFixtures: options.environmentFixtures,
+        manageTimeout: options.manageTimeout ?? false,
+        ...(options.workRoot ? { workRoot: options.workRoot } : {}),
+        ...(options.signal ? { signal: options.signal } : {}),
+        ...(options.onStage ? { onStage: options.onStage } : {}),
+    });
+}
+export async function runCodexEvalIsolationProbe(options) {
+    assertCodexEvalCanStartWork();
+    const sourceEnv = options.sourceEnv ?? process.env;
+    const processManagement = options.processManagement ?? DEFAULT_OBSERVED_PROCESS_EVIDENCE;
+    const temporaryRoot = await createEvalTemporaryRoot('isolation-probe-');
+    const workspaceRoot = path.join(temporaryRoot.root, 'project');
+    const codexHome = path.join(temporaryRoot.root, 'codex-home');
+    const control = new CodexEvalAttemptControl({
+        label: 'Codex eval isolation probe',
+        ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),
+        ...(options.signal ? { signal: options.signal } : {}),
+    });
+    let session;
+    let removeAbortHandler;
+    try {
+        await fs.copy(options.subjectRoot, workspaceRoot, { dereference: false });
+        const allowedSkillRoots = await isolateWorkspaceSkills({
+            workspaceRoot,
+            skillNames: options.skillNames ?? [options.skillName],
+        });
+        const env = buildIsolatedEnv(sourceEnv, temporaryRoot.root, codexHome, options.codexRuntime);
+        await Promise.all([
+            fs.ensureDir(env.HOME),
+            fs.ensureDir(env.TMPDIR),
+            materializeCodexEvalRuntimeHome(options.codexRuntime, codexHome, {
+                trustedProject: workspaceRoot,
+            }),
+        ]);
+        if (processManagement.mode === 'managed' || processManagement.mode === 'observed') {
+            await assertProcessTreeInspectionAvailable();
+        }
+        control.throwIfStopped('initialize');
+        session = new EvalAppServerSession({
+            codexBin: options.codexBin,
+            cwd: workspaceRoot,
+            env,
+            modelProvider: options.codexRuntime.modelProvider,
+            sensitiveEnvironmentNames: options.codexRuntime.sensitiveEnvironmentNames,
+            isolation: 'strict',
+            processManagement,
+        });
+        const abort = () => {
+            void session?.cleanup().catch(() => undefined);
+        };
+        if (control.signal.aborted)
+            abort();
+        else {
+            control.signal.addEventListener('abort', abort, { once: true });
+            removeAbortHandler = () => control.signal.removeEventListener('abort', abort);
+        }
+        await session.initialize(control.rpcTimeout('initialize'));
+        control.throwIfStopped('initialize');
+        const evidence = await collectStrictIsolationEvidence({
+            session,
+            workspaceRoot,
+            codexHome,
+            allowedSkillRoots,
+            control,
+        });
+        await session.cleanup();
+        control.throwIfStopped('cleanup');
+        return evidence;
+    }
+    finally {
+        removeAbortHandler?.();
+        try {
+            await session?.cleanup();
+        }
+        catch (error) {
+            throw new Error(`Codex eval isolation probe cleanup failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        finally {
+            try {
+                await temporaryRoot.cleanup();
+            }
+            finally {
+                control.dispose();
+            }
+        }
+    }
+}
+//# sourceMappingURL=codex-eval-app-server.js.map