@lesliechan721/agent-skill-eval 0.1.0-beta.5 → 0.1.0-beta.51

package/dist/codex-workflow-fixtures.js

@@ -0,0 +1,144 @@
+import { createHash } from 'node:crypto';
+import path from 'node:path';
+import fs from 'fs-extra';
+import { assertHostReadContainmentAvailable } from './codex-eval-app-server.js';
+import { createCodexEvalTemporaryRoot, redactCodexEvalValue } from './codex-eval-runtime-config.js';
+import { applyCodexEvalGitIsolation } from './codex-eval-isolation.js';
+import { runProviderProcess } from './process.js';
+function object(value, label) {
+    if (!value || typeof value !== 'object' || Array.isArray(value))
+        throw new Error(`${label} must be an object.`);
+    return value;
+}
+function exact(value, required, optional, label) {
+    if (required.some((key) => !Object.hasOwn(value, key)))
+        throw new Error(`${label} is missing a required field.`);
+    if (Object.keys(value).some((key) => ![...required, ...optional].includes(key)))
+        throw new Error(`${label} has an unknown field.`);
+}
+function relative(value) {
+    if (typeof value !== 'string' || !value || Buffer.byteLength(value) > 1_024 || value.includes('\\') || value.includes('\0')
+        || path.posix.isAbsolute(value) || value.split('/').some((segment) => !segment || segment === '.' || segment === '..')) {
+        throw new Error('workflow fixture asset must be a normalized relative path.');
+    }
+    return value;
+}
+function json(value, depth = 0) {
+    if (depth > 8)
+        throw new Error('workflow fixture JSON depth limit exceeded.');
+    if (value === null || typeof value === 'boolean' || typeof value === 'string')
+        return value;
+    if (typeof value === 'number' && Number.isFinite(value))
+        return value;
+    if (Array.isArray(value) && value.length <= 128)
+        return value.map((entry) => json(entry, depth + 1));
+    const entries = Object.entries(object(value, 'workflow fixture JSON'));
+    if (entries.length > 128)
+        throw new Error('workflow fixture JSON field limit exceeded.');
+    return Object.fromEntries(entries.map(([key, entry]) => [key, json(entry, depth + 1)]));
+}
+export function parseWorkflowFixtureAction(value) {
+    const action = object(value, 'workflow fixture action');
+    exact(action, ['command', 'input'], ['pathPrepend'], 'workflow fixture action');
+    if (!Array.isArray(action.command) || action.command.length !== 2 || action.command[0] !== 'node') {
+        throw new Error('workflow fixture command must be ["node", relative fixture script].');
+    }
+    const input = json(action.input);
+    if (Buffer.byteLength(JSON.stringify(input)) > 200_000)
+        throw new Error('workflow fixture input byte limit exceeded.');
+    let pathPrepend;
+    if (action.pathPrepend !== undefined) {
+        if (!Array.isArray(action.pathPrepend) || action.pathPrepend.length > 8)
+            throw new Error('workflow fixture pathPrepend has too many entries.');
+        pathPrepend = action.pathPrepend.map(relative);
+        if (new Set(pathPrepend).size !== pathPrepend.length)
+            throw new Error('workflow fixture pathPrepend must be unique.');
+    }
+    return { command: ['node', relative(action.command[1])], input, ...(pathPrepend ? { pathPrepend } : {}) };
+}
+async function fixtureAsset(root, relativePath, directory = false) {
+    let current = root;
+    for (const segment of relative(relativePath).split('/')) {
+        current = path.join(current, segment);
+        if ((await fs.lstat(current)).isSymbolicLink())
+            throw new Error('workflow fixture assets must not contain symbolic links.');
+    }
+    const info = await fs.lstat(current);
+    if (directory ? !info.isDirectory() : !info.isFile() || info.size > 1024 * 1024)
+        throw new Error('workflow fixture asset kind or size is invalid.');
+    if (directory) {
+        for (const entry of await fs.readdir(current)) {
+            const child = await fs.lstat(path.join(current, entry));
+            if (child.isSymbolicLink() || (!child.isDirectory() && !child.isFile()))
+                throw new Error('workflow fixture PATH tree contains an unsafe node.');
+            if (child.isDirectory())
+                await fixtureAsset(root, `${relativePath}/${entry}`, true);
+        }
+    }
+    return current;
+}
+export async function validateWorkflowFixtureActions(actions, fixtures) {
+    for (const action of actions) {
+        await fixtureAsset(fixtures, action.command[1]);
+        for (const directory of action.pathPrepend ?? [])
+            await fixtureAsset(fixtures, directory, true);
+    }
+}
+export async function runWorkflowFixtureAction(options) {
+    await validateWorkflowFixtureActions([options.action], options.fixtures);
+    const script = await fixtureAsset(options.fixtures, options.action.command[1]);
+    const scriptSha256 = createHash('sha256').update(await fs.readFile(script)).digest('hex');
+    const temporaryRoot = await createCodexEvalTemporaryRoot('workflow-fixture-');
+    const root = await fs.realpath(temporaryRoot.root);
+    try {
+        const environment = {};
+        for (const key of ['PATH', 'LANG', 'LC_ALL'])
+            if (options.sourceEnv[key] !== undefined)
+                environment[key] = options.sourceEnv[key];
+        environment.PATH = [...(options.action.pathPrepend ?? []).map((entry) => path.join(options.fixtures, entry)), environment.PATH ?? ''].join(path.delimiter);
+        const env = applyCodexEvalGitIsolation({ ...environment, HOME: path.join(root, 'home'), USERPROFILE: path.join(root, 'home'),
+            TMPDIR: path.join(root, 'tmp'), TEMP: path.join(root, 'tmp'), TMP: path.join(root, 'tmp'), NO_COLOR: '1' });
+        await Promise.all([fs.ensureDir(env.HOME), fs.ensureDir(env.TMPDIR)]);
+        const allowed = [options.workspace, options.fixtures, root, path.dirname(process.execPath), '/System', '/usr', '/bin', '/sbin', '/dev', '/Library/Developer/CommandLineTools'];
+        let profile = [
+            '(version 1)', '(allow default)',
+            // macOS executable startup needs the root directory; descendants remain restricted.
+            `(deny file-read-data (require-all (require-not (literal "/")) ${allowed.map((entry) => `(require-not (subpath ${JSON.stringify(entry)}))`).join(' ')}))`,
+            '(deny file-write*)',
+            '(allow file-write* (literal "/dev/null"))', '(allow file-read* (literal "/dev/null") (literal "/dev/urandom"))',
+            ...[options.workspace, root].map((entry) => `(allow file-write* (subpath ${JSON.stringify(entry)}))`),
+            `(deny file-read* (subpath ${JSON.stringify(options.output)}))`,
+        ].join('\n');
+        for (const entry of allowed) {
+            for (let ancestor = path.dirname(entry); ancestor !== path.dirname(ancestor); ancestor = path.dirname(ancestor)) {
+                profile += `\n(allow file-read-metadata (literal ${JSON.stringify(ancestor)}))`;
+            }
+        }
+        await assertHostReadContainmentAvailable(profile);
+        const result = await runProviderProcess({
+            executable: '/usr/bin/sandbox-exec', args: ['-p', profile, process.execPath, script], cwd: options.workspace, env,
+            stdin: JSON.stringify({ version: 1, input: options.action.input, paths: { workspace: options.workspace, fixtures: options.fixtures } }),
+            timeoutMs: options.timeoutMs, maxBufferBytes: 1024 * 1024, ...(options.signal ? { signal: options.signal } : {}),
+        }, {}, {
+            containment: options.processManagement.mode === 'unmanaged-confirmed' ? 'unmanaged-confirmed' : 'verified-observed',
+            ...(options.processManagement.mode === 'unmanaged-confirmed' ? {} : { processTree: options.processTree }),
+        });
+        if (result.exitCode !== 0 || result.signal || result.timedOut || result.cancelled || result.maxBufferExceeded
+            || (options.processManagement.mode !== 'unmanaged-confirmed' && !result.gradingEvidenceAccepted)) {
+            throw new Error('workflow fixture action did not finish with confirmed process cleanup; no answer or grade was released.');
+        }
+        const value = object(JSON.parse(result.stdout), 'workflow fixture receipt');
+        exact(value, ['version', 'completed', 'artifacts', 'evidence'], [], 'workflow fixture receipt');
+        if (value.version !== 1 || value.completed !== true || !Array.isArray(value.artifacts) || value.artifacts.length > 64) {
+            throw new Error('workflow fixture action returned an invalid factual receipt.');
+        }
+        const artifacts = value.artifacts.map(relative);
+        for (const artifact of artifacts)
+            await fixtureAsset(options.workspace, artifact);
+        return redactCodexEvalValue({ receipt: { version: 1, completed: true, artifacts, evidence: json(value.evidence) }, scriptSha256 }, options.runtime);
+    }
+    finally {
+        await temporaryRoot.cleanup();
+    }
+}
+//# sourceMappingURL=codex-workflow-fixtures.js.map

package/dist/codex-workflow-fixtures.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"codex-workflow-fixtures.js","sourceRoot":"","sources":["../src/codex-workflow-fixtures.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,IAAI,MAAM,WAAW,CAAA;AAE5B,OAAO,EAAE,MAAM,UAAU,CAAA;AAEzB,OAAO,EAAE,kCAAkC,EAA2C,MAAM,4BAA4B,CAAA;AACxH,OAAO,EAAE,4BAA4B,EAAE,oBAAoB,EAA+B,MAAM,gCAAgC,CAAA;AAChI,OAAO,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAA;AACtE,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAmBjD,SAAS,MAAM,CAAC,KAAc,EAAE,KAAa;IAC3C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,qBAAqB,CAAC,CAAA;IAC/G,OAAO,KAAgC,CAAA;AACzC,CAAC;AAED,SAAS,KAAK,CAAC,KAA8B,EAAE,QAAkB,EAAE,QAAkB,EAAE,KAAa;IAClG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,+BAA+B,CAAC,CAAA;IAChH,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,QAAQ,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,wBAAwB,CAAC,CAAA;AACpI,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,KAAK,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;WACtH,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,IAAI,OAAO,KAAK,GAAG,IAAI,OAAO,KAAK,IAAI,CAAC,EAAE,CAAC;QACzH,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAA;IAC/E,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,IAAI,CAAC,KAAc,EAAE,KAAK,GAAG,CAAC;IACrC,IAAI,KAAK,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAA;IAC7E,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IAC3F,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IACrE,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAA;IACpG,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,uBAAuB,CAAC,CAAC,CAAA;IACtE,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAA;IACxF,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;AACzF,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,KAAc;IACvD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,EAAE,yBAAyB,CAAC,CAAA;IACvD,KAAK,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,CAAC,aAAa,CAAC,EAAE,yBAAyB,CAAC,CAAA;IAC/E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC;QAClG,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAA;IACxF,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IAChC,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,GAAG,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAA;IACtH,IAAI,WAAiC,CAAA;IACrC,IAAI,MAAM,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAA;QAC9I,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAC9C,IAAI,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAA;IACvH,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAA;AAC3G,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,YAAoB,EAAE,SAAS,GAAG,KAAK;IAC/E,IAAI,OAAO,GAAG,IAAI,CAAA;IAClB,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACxD,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QACrC,IAAI,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAA;IAC7H,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;IACpC,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAA;IACnJ,IAAI,SAAS,EAAE,CAAC;QACd,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAA;YACvD,IAAI,KAAK,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAA;YAC/I,IAAI,KAAK,CAAC,WAAW,EAAE;gBAAE,MAAM,YAAY,CAAC,IAAI,EAAE,GAAG,YAAY,IAAI,KAAK,EAAE,EAAE,IAAI,CAAC,CAAA;QACrF,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAAC,OAAgC,EAAE,QAAgB;IACrG,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;QAC/C,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,WAAW,IAAI,EAAE;YAAE,MAAM,YAAY,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,CAAA;IACjG,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,OAa9C;IACC,MAAM,8BAA8B,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAA;IACxE,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;IAC9E,MAAM,YAAY,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IACzF,MAAM,aAAa,GAAG,MAAM,4BAA4B,CAAC,mBAAmB,CAAC,CAAA;IAC7E,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,CAAA;IAClD,IAAI,CAAC;QACH,MAAM,WAAW,GAAsB,EAAE,CAAA;QACzC,KAAK,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAU;YAAE,IAAI,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,SAAS;gBAAE,WAAW,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;QAC1I,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAC1J,MAAM,GAAG,GAAG,0BAA0B,CAAC,EAAE,GAAG,WAAW,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC;YAC1H,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAA;QAC7G,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,IAAK,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,MAAO,CAAC,CAAC,CAAC,CAAA;QACvE,MAAM,OAAO,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,qCAAqC,CAAC,CAAA;QAC9K,IAAI,OAAO,GAAG;YACZ,aAAa,EAAE,iBAAiB;YAChC,oFAAoF;YACpF,iEAAiE,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,yBAAyB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI;YACzJ,oBAAoB;YACpB,2CAA2C,EAAE,mEAAmE;YAChH,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,+BAA+B,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC;YACrG,6BAA6B,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI;SAChE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACZ,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,KAAK,IAAI,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,QAAQ,KAAK,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAChH,OAAO,IAAI,wCAAwC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAA;YACjF,CAAC;QACH,CAAC;QACD,MAAM,kCAAkC,CAAC,OAAO,CAAC,CAAA;QACjD,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC;YACtC,UAAU,EAAE,uBAAuB,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,SAAS,EAAE,GAAG;YACjH,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;YACvI,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,cAAc,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACjH,EAAE,EAAE,EAAE;YACL,WAAW,EAAE,OAAO,CAAC,iBAAiB,CAAC,IAAI,KAAK,qBAAqB,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,mBAAmB;YACnH,GAAG,CAAC,OAAO,CAAC,iBAAiB,CAAC,IAAI,KAAK,qBAAqB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC;SAC1G,CAAC,CAAA;QACF,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,iBAAiB;eACxG,CAAC,OAAO,CAAC,iBAAiB,CAAC,IAAI,KAAK,qBAAqB,IAAI,CAAC,MAAM,CAAC,uBAAuB,CAAC,EAAE,CAAC;YACnG,MAAM,IAAI,KAAK,CAAC,yGAAyG,CAAC,CAAA;QAC5H,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,0BAA0B,CAAC,CAAA;QAC3E,KAAK,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,CAAC,EAAE,EAAE,EAAE,0BAA0B,CAAC,CAAA;QAC/F,IAAI,KAAK,CAAC,OAAO,KAAK,CAAC,IAAI,KAAK,CAAC,SAAS,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YACtH,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAA;QACjF,CAAC;QACD,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAC/C,KAAK,MAAM,QAAQ,IAAI,SAAS;YAAE,MAAM,YAAY,CAAC,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;QACjF,OAAO,oBAAoB,CAAC,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;IACrJ,CAAC;YAAS,CAAC;QACT,MAAM,aAAa,CAAC,OAAO,EAAE,CAAA;IAC/B,CAAC;AACH,CAAC"}

package/dist/codex-workflow.d.ts

@@ -0,0 +1,224 @@
+import { type CodexEvalProcessManagementEvidence } from './codex-eval-app-server.js';
+import { type CodexEvalRuntimeConfig } from './codex-eval-runtime-config.js';
+import { type InstructionV2ReasoningEffort, type InstructionV2Receipt, type InstructionV2Transport } from './instruction-v2.js';
+import type { ProcessTreeDependencies } from './process-tree.js';
+import { type EvalProviderRunInput, type EvalProviderRunResult } from './custom-process.js';
+import { type WorkflowEvaluationRequest } from './codex-workflow-evaluation.js';
+export declare const CODEX_WORKFLOW_ARTIFACTS: readonly [{
+    readonly path: "workflow-evidence.json";
+    readonly role: "evidence";
+}, {
+    readonly path: "workflow-trace.json";
+    readonly role: "trace";
+}, {
+    readonly path: "model-output.txt";
+    readonly role: "output";
+}];
+export interface CodexWorkflowProvider {
+    type: 'codex-workflow';
+    command: readonly [string, ...string[]];
+    allowedModels: readonly string[];
+}
+export interface CodexWorkflowRole {
+    name: string;
+    description: string;
+    configFile: string;
+}
+export interface CodexWorkflowRequest {
+    version: 1;
+    prompt: string;
+    sandbox: 'read-only' | 'workspace-write';
+    model: string | null;
+    reasoningEffort?: InstructionV2ReasoningEffort;
+    developerInstructions: string;
+    roles: CodexWorkflowRole[];
+    evaluation?: WorkflowEvaluationRequest;
+    transport?: InstructionV2Transport;
+    multiAgent: {
+        protocol: 'v1' | 'v2';
+        maxDepth: number;
+        hideSpawnAgentMetadata: boolean;
+        exposeSpawnAgentModelOverrides: boolean;
+    };
+}
+export interface CodexWorkflowArtifact {
+    path: string;
+    role: string;
+}
+export interface CodexWorkflowAdapterInput {
+    version: 1;
+    phase: 'prepare' | 'grade';
+    case: Record<string, unknown>;
+    variant: EvalProviderRunInput['variant'];
+    attempt: number;
+    paths: {
+        workspace: string;
+        fixtures: string;
+        output: string;
+        evidence?: string;
+        modelOutput?: string;
+    };
+}
+export interface CodexWorkflowCall {
+    ordinal: number;
+    threadId: string;
+    parentThreadId: string | null;
+    linkedThreadId?: string;
+    callId: string | null;
+    namespace: string | null;
+    tool: string;
+    arguments: unknown;
+    effectiveRuntime?: {
+        model?: string;
+        reasoningEffort?: string;
+    };
+    output: unknown;
+    status: string | null;
+}
+export interface CodexWorkflowEvidence {
+    version: 1;
+    rootThreadId: string;
+    completion: {
+        status: 'completed' | 'failed' | 'cancelled' | 'unknown';
+        turnStatus: string;
+        durationMs: number;
+    };
+    termination: {
+        reason: 'completed' | 'failed' | 'cancelled' | 'unknown';
+        appServerProcessTreeCleanupConfirmed?: true;
+        providerProcessCleanupConfirmed?: boolean;
+    };
+    usage: unknown;
+    finalMessage: string;
+    workspaceDiff: {
+        added: string[];
+        removed: string[];
+        modified: string[];
+    };
+    skillLoads: Array<{
+        threadId: string;
+        skill: string;
+        callId: string | null;
+    }>;
+    threadGraph: Array<{
+        threadId: string;
+        parentThreadId: string | null;
+        agentRole: string | null;
+        initialMessage: {
+            kind: 'plaintext';
+            text: string;
+        } | {
+            kind: 'opaque-link';
+            sha256: string;
+            linkedCallId: string;
+        } | null;
+        developerContext: {
+            present: boolean;
+            messageCount: number;
+            sha256: string | null;
+            hasMultiAgentModeGate: boolean;
+            hasNestedAgentCapability: boolean;
+            hasCodexExploreDeveloperAuthorization: boolean;
+        };
+    }>;
+    collaborationCalls: CodexWorkflowCall[];
+    leafCalls: CodexWorkflowCall[];
+    cleanup: {
+        isolatedCodexHomeRemoved: true;
+    };
+    evaluation?: Record<string, unknown>;
+    launch?: {
+        instructionChannel: 'developer';
+        developerInstructions: string;
+        instructionsRevision: string;
+        model: string | null;
+        reasoningEffort: InstructionV2ReasoningEffort | null;
+        sandbox: CodexWorkflowRequest['sandbox'];
+        roles: CodexWorkflowRole[];
+        configOverrides: string[];
+        hostRoleAdoption: 'unknown';
+    };
+    transport?: {
+        type: 'instruction-v2';
+        request: InstructionV2Transport['request'];
+        command: readonly string[];
+        receipt: InstructionV2Receipt | null;
+        traceAvailable: boolean;
+        observationLimits: string[];
+        diagnostic: string | null;
+        exitCode: number | null;
+        signal: NodeJS.Signals | null;
+        stderr: string;
+    };
+}
+export interface CodexWorkflowExecutionResult {
+    evidence: CodexWorkflowEvidence;
+    trace: unknown[];
+    modelOutput: string;
+    usage: unknown;
+}
+export interface CodexWorkflowTransportInput {
+    request: CodexWorkflowRequest;
+    input: EvalProviderRunInput;
+    evalRoot: string;
+    codexRuntime: CodexEvalRuntimeConfig;
+    codexBin: string;
+    sourceEnv: NodeJS.ProcessEnv;
+    timeoutMs: number;
+    processManagement: CodexEvalProcessManagementEvidence;
+    processTree?: ProcessTreeDependencies;
+    protectedEvalRoots?: readonly string[];
+    signal?: AbortSignal;
+}
+export interface CodexWorkflowTransport {
+    run(input: CodexWorkflowTransportInput): Promise<CodexWorkflowExecutionResult>;
+}
+export interface CodexWorkflowExecutorOptions {
+    provider: CodexWorkflowProvider;
+    evalRoot: string;
+    input: EvalProviderRunInput;
+    codexRuntime: CodexEvalRuntimeConfig;
+    sourceEnv?: NodeJS.ProcessEnv;
+    codexBin?: string;
+    platform?: NodeJS.Platform;
+    processTree?: ProcessTreeDependencies;
+    processManagement?: CodexEvalProcessManagementEvidence;
+    transport?: CodexWorkflowTransport;
+    protectedEvalRoots?: readonly string[];
+}
+export declare function parseCodexWorkflowRequest(value: unknown, allowedModels: readonly string[], workspace: string, fixtures?: string): Promise<CodexWorkflowRequest>;
+export declare function codexWorkflowAdapterEnvironment(sourceEnv: NodeJS.ProcessEnv, evalRoot: string, phase: 'prepare' | 'grade'): NodeJS.ProcessEnv;
+export declare function codexWorkflowFrameThreadId(frame: Record<string, unknown>): string | null;
+export declare function codexWorkflowFrameIsForRoot(frame: Record<string, unknown>, rootThreadId: string): boolean;
+export declare function codexWorkflowMissingFinalMessageDiagnostic(options: {
+    frames: readonly unknown[];
+    rootThreadId: string;
+    turnStatus: string | null;
+    usage: unknown;
+}): {
+    rootTurnStatus: string | null;
+    rootFinalAgentMessageCount: number;
+    otherThreadFinalAgentMessageCount: number;
+    usagePresent: boolean;
+    frameCount: number;
+};
+export declare function normalizeCodexWorkflowRollouts(options: {
+    codexHome: string;
+    rootThreadId: string;
+    request: CodexWorkflowRequest;
+    runtime: CodexEvalRuntimeConfig;
+    appServerFrames?: readonly unknown[];
+    isolatedPaths?: readonly {
+        path: string;
+        marker: string;
+    }[];
+}): Promise<{
+    threadGraph: CodexWorkflowEvidence['threadGraph'];
+    collaborationCalls: CodexWorkflowCall[];
+    leafCalls: CodexWorkflowCall[];
+    skillLoads: CodexWorkflowEvidence['skillLoads'];
+    trace: unknown[];
+}>;
+export declare function executeCodexWorkflowProvider(options: CodexWorkflowExecutorOptions): Promise<EvalProviderRunResult & {
+    artifacts: CodexWorkflowArtifact[];
+}>;