@lesliechan721/agent-skill-eval 0.1.0-beta.8 → 0.1.0-beta.9

package/dist/bin.js

@@ -1,80 +1,190 @@
 #!/usr/bin/env node
 import path from 'node:path';
 import process from 'node:process';
-import { computeSkillDigest, getRuntimeCapabilities, getSchema, runExecutionPlan, validateExecutionPlan, validateRunBundle, } from './index.js';
-import { isSchemaName } from './schemas.js';
+import { createEvalResultBinding, doctorEvalProviders, runEvalSpec, validateEvalAuditFile, validateEvalResultFile, validateEvalSpecFile, validateEvalProviderSuiteKind, validateSelectedCasesFile, } from './index.js';
 const help = `Usage: aw-skill-eval <command>
 
 Commands:
-  capabilities --json
-  schema print <aw-skill-eval-execution-plan|eval-run-bundle>
-  plan validate <plan.json>
-  run <plan.json>
-  bundle validate <bundle.json>
-  digest skill <skill-dir>
+  spec validate <spec.json>
+  audit validate <audit.json> --spec <spec.json>
+  run <spec.json> --audit <audit.json>
+  result validate <result.json> --spec <spec.json> --audit <audit.json>
+  inspect-result <result.json> --spec <spec.json> --audit <audit.json>
+  doctor --json
 `;
 function requirePath(value, usage) {
     if (!value)
         throw new Error(usage);
     return path.resolve(value);
 }
+function optionPath(argv, name, usage) {
+    const index = argv.indexOf(name);
+    if (index < 0 || !argv[index + 1] || argv.indexOf(name, index + 1) >= 0)
+        throw new Error(usage);
+    return requirePath(argv[index + 1], usage);
+}
+async function boundSpec(specPath) {
+    const validation = await validateEvalSpecFile(specPath);
+    if (!validation.valid || !validation.calculated) {
+        throw new Error(`EvalSpec is invalid: ${validation.errors.join('; ')}`);
+    }
+    const cases = await validateSelectedCasesFile(validation.calculated.paths.cases);
+    if (!cases.valid || !cases.calculated) {
+        throw new Error(`Selected cases are invalid: ${cases.errors.join('; ')}`);
+    }
+    if (cases.calculated.suite.unitUnderTest !== validation.calculated.spec.source.skill) {
+        throw new Error('Selected cases unitUnderTest does not match EvalSpec source.skill.');
+    }
+    const providerErrors = validateEvalProviderSuiteKind(validation.calculated.spec.provider, cases.calculated.suite.kind);
+    if (providerErrors.length > 0)
+        throw new Error(providerErrors.join('; '));
+    return { spec: validation.calculated, cases: cases.calculated };
+}
+async function boundResult(specPath, auditPath) {
+    const binding = await boundSpec(specPath);
+    const audit = await validateEvalAuditFile(auditPath, {
+        spec: binding.spec.spec,
+        selectedCaseIds: binding.cases.cases.map((entry) => entry.id),
+    });
+    if (!audit.valid || !audit.calculated) {
+        throw new Error(`EvalAudit is invalid: ${audit.errors.join('; ')}`);
+    }
+    return createEvalResultBinding(binding.spec, audit.calculated.auditDigest, binding.cases);
+}
 async function main(argv) {
     if (argv.length === 0 || argv[0] === '--help' || argv[0] === '-h') {
         process.stdout.write(help);
         return;
     }
     const [command, action, value, ...extra] = argv;
-    if (extra.length > 0)
-        throw new Error('Too many arguments. Use --help for usage.');
-    if (command === 'capabilities') {
-        if (action !== '--json' || value !== undefined)
-            throw new Error('Usage: aw-skill-eval capabilities --json');
-        process.stdout.write(`${JSON.stringify(getRuntimeCapabilities(), null, 2)}\n`);
-        return;
-    }
-    if (command === 'schema') {
-        if (action !== 'print' || !value || !isSchemaName(value) || value === 'eval-runtime-capabilities')
-            throw new Error('Usage: aw-skill-eval schema print <aw-skill-eval-execution-plan|eval-run-bundle>');
-        process.stdout.write(`${JSON.stringify(await getSchema(value), null, 2)}\n`);
+    if (command === 'doctor') {
+        if (action !== '--json' || value !== undefined || extra.length > 0) {
+            throw new Error('Usage: aw-skill-eval doctor --json');
+        }
+        process.stdout.write(`${JSON.stringify(await doctorEvalProviders(), null, 2)}\n`);
         return;
     }
-    if (command === 'plan') {
-        if (action !== 'validate')
-            throw new Error('Usage: aw-skill-eval plan validate <plan.json>');
-        const result = await validateExecutionPlan(requirePath(value, 'Usage: aw-skill-eval plan validate <plan.json>'));
+    if (command === 'spec') {
+        if (action !== 'validate' || !value || extra.length > 0) {
+            throw new Error('Usage: aw-skill-eval spec validate <spec.json>');
+        }
+        const specPath = requirePath(value, 'Usage: aw-skill-eval spec validate <spec.json>');
+        const validation = await validateEvalSpecFile(specPath);
+        const cases = validation.valid && validation.calculated
+            ? await validateSelectedCasesFile(validation.calculated.paths.cases)
+            : null;
+        const errors = [...validation.errors, ...(cases?.errors ?? [])];
+        const unitMatches = !validation.calculated || !cases?.calculated ||
+            cases.calculated.suite.unitUnderTest === validation.calculated.spec.source.skill;
+        if (!unitMatches)
+            errors.push('Selected cases unitUnderTest does not match EvalSpec source.skill.');
+        if (validation.calculated && cases?.calculated) {
+            errors.push(...validateEvalProviderSuiteKind(validation.calculated.spec.provider, cases.calculated.suite.kind));
+        }
+        const result = {
+            valid: errors.length === 0,
+            errors,
+            calculated: errors.length === 0 ? validation.calculated : null,
+        };
         process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
         if (!result.valid)
             process.exitCode = 1;
         return;
     }
     if (command === 'run') {
-        if (value !== undefined)
-            throw new Error('Usage: aw-skill-eval run <plan.json>');
-        const result = await runExecutionPlan(requirePath(action, 'Usage: aw-skill-eval run <plan.json>'));
-        process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
-        if (!result.requirementsMet)
-            process.exitCode = 1;
+        const runPath = requirePath(action, 'Usage: aw-skill-eval run <spec.json> --audit <audit.json>');
+        if (value !== '--audit' || extra.length !== 1) {
+            throw new Error('Usage: aw-skill-eval run <spec.json> --audit <audit.json>');
+        }
+        const controller = new AbortController();
+        let receivedSignal = null;
+        const handleSignal = (signal) => {
+            if (receivedSignal)
+                return;
+            receivedSignal = signal;
+            controller.abort();
+        };
+        const handleSigint = () => handleSignal('SIGINT');
+        const handleSigterm = () => handleSignal('SIGTERM');
+        process.on('SIGINT', handleSigint);
+        process.on('SIGTERM', handleSigterm);
+        try {
+            const result = await runEvalSpec(runPath, {
+                auditPath: requirePath(extra[0], '--audit requires a path'),
+                signal: controller.signal,
+            });
+            process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+        }
+        finally {
+            process.off('SIGINT', handleSigint);
+            process.off('SIGTERM', handleSigterm);
+            if (receivedSignal)
+                process.exitCode = receivedSignal === 'SIGINT' ? 130 : 143;
+        }
         return;
     }
-    if (command === 'bundle') {
-        if (action !== 'validate')
-            throw new Error('Usage: aw-skill-eval bundle validate <bundle.json>');
-        const result = await validateRunBundle(requirePath(value, 'Usage: aw-skill-eval bundle validate <bundle.json>'));
+    if (command === 'result') {
+        if (action !== 'validate' || !value || extra.length !== 4) {
+            throw new Error('Usage: aw-skill-eval result validate <result.json> --spec <spec.json> --audit <audit.json>');
+        }
+        const specPath = optionPath(extra, '--spec', 'result validate requires one --spec path');
+        const auditPath = optionPath(extra, '--audit', 'result validate requires one --audit path');
+        const binding = await boundResult(specPath, auditPath);
+        const result = await validateEvalResultFile(requirePath(value, 'result validate requires a result path'), binding);
         process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
         if (!result.valid)
             process.exitCode = 1;
         return;
     }
-    if (command === 'digest') {
-        if (action !== 'skill')
-            throw new Error('Usage: aw-skill-eval digest skill <skill-dir>');
-        process.stdout.write(`${await computeSkillDigest(requirePath(value, 'Usage: aw-skill-eval digest skill <skill-dir>'))}\n`);
+    if (command === 'inspect-result') {
+        const options = value === undefined ? [] : [value, ...extra];
+        if (!action || options.length !== 4) {
+            throw new Error('Usage: aw-skill-eval inspect-result <result.json> --spec <spec.json> --audit <audit.json>');
+        }
+        const binding = await boundResult(optionPath(options, '--spec', 'inspect-result requires one --spec path'), optionPath(options, '--audit', 'inspect-result requires one --audit path'));
+        const validation = await validateEvalResultFile(requirePath(action, 'inspect-result requires a result path'), binding);
+        if (!validation.valid || !validation.calculated) {
+            process.stdout.write(`${JSON.stringify(validation, null, 2)}\n`);
+            process.exitCode = 1;
+            return;
+        }
+        const groups = new Map();
+        for (const run of validation.calculated.result.runs) {
+            const key = `${run.caseId}\0${run.attempt}`;
+            const group = groups.get(key) ?? { caseId: run.caseId, attempt: run.attempt, variants: {} };
+            group.variants[run.variant] = {
+                status: run.status,
+                durationMs: run.durationMs,
+                outputDir: run.outputDir,
+                usage: run.usage,
+                grade: run.grade,
+                error: run.error,
+            };
+            groups.set(key, group);
+        }
+        process.stdout.write(`${JSON.stringify({
+            version: 1,
+            summary: validation.calculated.result.summary,
+            groups: [...groups.values()],
+        }, null, 2)}\n`);
+        return;
+    }
+    if (command === 'audit' && action === 'validate' && value && extra.length === 2 && extra[0] === '--spec') {
+        const binding = await boundSpec(requirePath(extra[1], 'audit validate requires a --spec path'));
+        const result = await validateEvalAuditFile(requirePath(value, 'audit validate requires an audit path'), {
+            spec: binding.spec.spec,
+            selectedCaseIds: binding.cases.cases.map((entry) => entry.id),
+        });
+        process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+        if (!result.valid)
+            process.exitCode = 1;
         return;
     }
     throw new Error(`Unknown command: ${command}. Use --help for usage.`);
 }
 main(process.argv.slice(2)).catch((error) => {
     process.stderr.write(`[ERROR] ${error.message}\n`);
-    process.exitCode = 1;
+    if (!process.exitCode)
+        process.exitCode = 1;
 });
 //# sourceMappingURL=bin.js.map

package/dist/bin.js.map

@@ -1 +1 @@
-{"version":3,"file":"bin.js","sourceRoot":"","sources":["../src/bin.ts"],"names":[],"mappings":";AACA,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,OAAO,MAAM,cAAc,CAAA;AAClC,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACtB,SAAS,EACT,gBAAgB,EAChB,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAE3C,MAAM,IAAI,GAAG;;;;;;;;;CASZ,CAAA;AAED,SAAS,WAAW,CAAC,KAAyB,EAAE,KAAa;IAC3D,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,CAAA;IAClC,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;AAC5B,CAAC;AAED,KAAK,UAAU,IAAI,CAAC,IAAc;IAChC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAClE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAC1B,OAAM;IACR,CAAC;IACD,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC,GAAG,IAAI,CAAA;IAC/C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;IAClF,IAAI,OAAO,KAAK,cAAc,EAAE,CAAC;QAC/B,IAAI,MAAM,KAAK,QAAQ,IAAI,KAAK,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAA;QAC3G,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,sBAAsB,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;QAC9E,OAAM;IACR,CAAC;IACD,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,IAAI,MAAM,KAAK,OAAO,IAAI,CAAC,KAAK,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,2BAA2B;YAC/F,MAAM,IAAI,KAAK,CAAC,kFAAkF,CAAC,CAAA;QACrG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,SAAS,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;QAC5E,OAAM;IACR,CAAC;IACD,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;QACvB,IAAI,MAAM,KAAK,UAAU;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAA;QAC5F,MAAM,MAAM,GAAG,MAAM,qBAAqB,CAAC,WAAW,CAAC,KAAK,EAAE,gDAAgD,CAAC,CAAC,CAAA;QAChH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,MAAM,CAAC,KAAK;YAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;QACvC,OAAM;IACR,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,EAAE,CAAC;QACtB,IAAI,KAAK,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;QAChF,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,MAAM,EAAE,sCAAsC,CAAC,CAAC,CAAA;QAClG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,MAAM,CAAC,eAAe;YAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;QACjD,OAAM;IACR,CAAC;IACD,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,IAAI,MAAM,KAAK,UAAU;YAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAA;QAChG,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,WAAW,CAAC,KAAK,EAAE,oDAAoD,CAAC,CAAC,CAAA;QAChH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,MAAM,CAAC,KAAK;YAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;QACvC,OAAM;IACR,CAAC;IACD,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,IAAI,MAAM,KAAK,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAA;QACxF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,kBAAkB,CAAC,WAAW,CAAC,KAAK,EAAE,+CAA+C,CAAC,CAAC,IAAI,CAAC,CAAA;QAC1H,OAAM;IACR,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,yBAAyB,CAAC,CAAA;AACvE,CAAC;AAED,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IACnD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAY,KAAe,CAAC,OAAO,IAAI,CAAC,CAAA;IAC7D,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;AACtB,CAAC,CAAC,CAAA"}
+{"version":3,"file":"bin.js","sourceRoot":"","sources":["../src/bin.ts"],"names":[],"mappings":";AACA,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,OAAO,MAAM,cAAc,CAAA;AAClC,OAAO,EACL,uBAAuB,EACvB,mBAAmB,EACnB,WAAW,EACX,qBAAqB,EACrB,sBAAsB,EACtB,oBAAoB,EACpB,6BAA6B,EAC7B,yBAAyB,GAC1B,MAAM,YAAY,CAAA;AAEnB,MAAM,IAAI,GAAG;;;;;;;;;CASZ,CAAA;AAED,SAAS,WAAW,CAAC,KAAyB,EAAE,KAAa;IAC3D,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,CAAA;IAClC,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;AAC5B,CAAC;AAED,SAAS,UAAU,CAAC,IAAc,EAAE,IAAY,EAAE,KAAa;IAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IAChC,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,CAAA;IAC/F,OAAO,WAAW,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAA;AAC5C,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAgB;IAIvC,MAAM,UAAU,GAAG,MAAM,oBAAoB,CAAC,QAAQ,CAAC,CAAA;IACvD,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,wBAAwB,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACzE,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,yBAAyB,CAAC,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IAChF,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,+BAA+B,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC3E,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,aAAa,KAAK,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACrF,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAA;IACvF,CAAC;IACD,MAAM,cAAc,GAAG,6BAA6B,CAClD,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EACnC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAC5B,CAAA;IACD,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;IACzE,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,UAAU,EAAE,CAAA;AACjE,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,QAAgB,EAAE,SAAiB;IAC5D,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,CAAA;IACzC,MAAM,KAAK,GAAG,MAAM,qBAAqB,CAAC,SAAS,EAAE;QACnD,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI;QACvB,eAAe,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;KAC9D,CAAC,CAAA;IACF,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,yBAAyB,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACrE,CAAC;IACD,OAAO,uBAAuB,CAC5B,OAAO,CAAC,IAAI,EACZ,KAAK,CAAC,UAAU,CAAC,WAAW,EAC5B,OAAO,CAAC,KAAK,CACd,CAAA;AACH,CAAC;AAED,KAAK,UAAU,IAAI,CAAC,IAAc;IAChC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAClE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAC1B,OAAM;IACR,CAAC;IACD,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC,GAAG,IAAI,CAAA;IAC/C,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,IAAI,MAAM,KAAK,QAAQ,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QACvD,CAAC;QACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,mBAAmB,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;QACjF,OAAM;IACR,CAAC;IACD,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;QACvB,IAAI,MAAM,KAAK,UAAU,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAA;QACnE,CAAC;QACD,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,EAAE,gDAAgD,CAAC,CAAA;QACrF,MAAM,UAAU,GAAG,MAAM,oBAAoB,CAAC,QAAQ,CAAC,CAAA;QACvD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,IAAI,UAAU,CAAC,UAAU;YACrD,CAAC,CAAC,MAAM,yBAAyB,CAAC,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC;YACpE,CAAC,CAAC,IAAI,CAAA;QACR,MAAM,MAAM,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC,CAAA;QAC/D,MAAM,WAAW,GAAG,CAAC,UAAU,CAAC,UAAU,IAAI,CAAC,KAAK,EAAE,UAAU;YAC9D,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,aAAa,KAAK,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;QAClF,IAAI,CAAC,WAAW;YAAE,MAAM,CAAC,IAAI,CAAC,oEAAoE,CAAC,CAAA;QACnG,IAAI,UAAU,CAAC,UAAU,IAAI,KAAK,EAAE,UAAU,EAAE,CAAC;YAC/C,MAAM,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAC1C,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EACnC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAC5B,CAAC,CAAA;QACJ,CAAC;QACD,MAAM,MAAM,GAAG;YACb,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;YAC1B,MAAM;YACN,UAAU,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI;SAC/D,CAAA;QACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,MAAM,CAAC,KAAK;YAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;QACvC,OAAM;IACR,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,EAAE,CAAC;QACtB,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,EAAE,2DAA2D,CAAC,CAAA;QAChG,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAA;QAC9E,CAAC;QACD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;QACxC,IAAI,cAAc,GAA0B,IAAI,CAAA;QAChD,MAAM,YAAY,GAAG,CAAC,MAAsB,EAAQ,EAAE;YACpD,IAAI,cAAc;gBAAE,OAAM;YAC1B,cAAc,GAAG,MAAM,CAAA;YACvB,UAAU,CAAC,KAAK,EAAE,CAAA;QACpB,CAAC,CAAA;QACD,MAAM,YAAY,GAAG,GAAS,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAA;QACvD,MAAM,aAAa,GAAG,GAAS,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,CAAA;QACzD,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAA;QAClC,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,aAAa,CAAC,CAAA;QACpC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE;gBACxC,SAAS,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,yBAAyB,CAAC;gBAC3D,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAA;YACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;QAC9D,CAAC;gBAAS,CAAC;YACT,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAA;YACnC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,aAAa,CAAC,CAAA;YACrC,IAAI,cAAc;gBAAE,OAAO,CAAC,QAAQ,GAAG,cAAc,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAA;QAChF,CAAC;QACD,OAAM;IACR,CAAC;IACD,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,IAAI,MAAM,KAAK,UAAU,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1D,MAAM,IAAI,KAAK,CAAC,4FAA4F,CAAC,CAAA;QAC/G,CAAC;QACD,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,EAAE,QAAQ,EAAE,0CAA0C,CAAC,CAAA;QACxF,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,2CAA2C,CAAC,CAAA;QAC3F,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAA;QACtD,MAAM,MAAM,GAAG,MAAM,sBAAsB,CACzC,WAAW,CAAC,KAAK,EAAE,wCAAwC,CAAC,EAC5D,OAAO,CACR,CAAA;QACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,MAAM,CAAC,KAAK;YAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;QACvC,OAAM;IACR,CAAC;IACD,IAAI,OAAO,KAAK,gBAAgB,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,KAAK,CAAC,CAAA;QAC5D,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,2FAA2F,CAAC,CAAA;QAC9G,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,WAAW,CAC/B,UAAU,CAAC,OAAO,EAAE,QAAQ,EAAE,yCAAyC,CAAC,EACxE,UAAU,CAAC,OAAO,EAAE,SAAS,EAAE,0CAA0C,CAAC,CAC3E,CAAA;QACD,MAAM,UAAU,GAAG,MAAM,sBAAsB,CAC7C,WAAW,CAAC,MAAM,EAAE,uCAAuC,CAAC,EAC5D,OAAO,CACR,CAAA;QACD,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;YAChD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;YAChE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;YACpB,OAAM;QACR,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,GAAG,EAIlB,CAAA;QACJ,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpD,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,MAAM,KAAK,GAAG,CAAC,OAAO,EAAE,CAAA;YAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAA;YAC3F,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG;gBAC5B,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,KAAK,EAAE,GAAG,CAAC,KAAK;aACjB,CAAA;YACD,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACxB,CAAC;QACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC;YACrC,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO;YAC7C,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;SAC7B,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;QAChB,OAAM;IACR,CAAC;IACD,IAAI,OAAO,KAAK,OAAO,IAAI,MAAM,KAAK,UAAU,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;QACzG,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,uCAAuC,CAAC,CAAC,CAAA;QAC/F,MAAM,MAAM,GAAG,MAAM,qBAAqB,CAAC,WAAW,CAAC,KAAK,EAAE,uCAAuC,CAAC,EAAE;YACtG,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI;YACvB,eAAe,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;SAC9D,CAAC,CAAA;QACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,MAAM,CAAC,KAAK;YAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;QACvC,OAAM;IACR,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,yBAAyB,CAAC,CAAA;AACvE,CAAC;AAED,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IACnD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAY,KAAe,CAAC,OAAO,IAAI,CAAC,CAAA;IAC7D,IAAI,CAAC,OAAO,CAAC,QAAQ;QAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;AAC7C,CAAC,CAAC,CAAA"}

package/dist/codex-eval-app-server.d.ts

@@ -0,0 +1,199 @@
+import { type ChildProcess } from 'node:child_process';
+import { type ChildProcessTreeMonitor } from './codex-eval-process-tree.js';
+import { type CodexEvalRuntimeConfig } from './codex-eval-runtime-config.js';
+import { type CodexEvalRpcIsolationEvidence } from './codex-eval-isolation.js';
+import { type AwSkillEvalIsolation } from './codex-eval-cases.js';
+export interface CodexEvalProcessManagementEvidence {
+    mode: 'managed' | 'observed' | 'unmanaged-confirmed' | 'not-applicable';
+    inspectionAvailable: boolean;
+    userAcknowledged: boolean;
+    descendantCleanupGuaranteed: boolean;
+    limitation?: string;
+    diagnostic?: string;
+}
+interface ScriptedReply {
+    expectedQuestionCount?: number;
+    answersByIndex: string[];
+    unexpectedQuestionAnswer: string;
+}
+interface WorkspaceArtifactExpectation {
+    path: string;
+    exists: boolean;
+    contains: string[];
+    notContains: string[];
+    json?: Record<string, unknown>;
+}
+interface WorkspaceArtifactEvidence {
+    path: string;
+    exists: boolean;
+    kind: 'absent' | 'file' | 'other';
+    bytes?: number;
+    sha256?: string;
+    text?: string;
+}
+export interface ConversationEvalCase {
+    id: string;
+    prompt: string;
+    scriptedReplies: ScriptedReply[];
+    expectedOutput: string;
+    checks: string[];
+    workspaceFiles: Record<string, string>;
+    workspaceUnchanged: boolean;
+    sandbox: 'read-only' | 'workspace-write';
+    allowedWorkspaceChanges: string[];
+    artifacts: WorkspaceArtifactExpectation[];
+    environmentPathPrepend: string[];
+}
+export interface AppServerRunEvidence {
+    variant: 'with_skill' | 'without_skill';
+    complete: boolean;
+    mechanicalPassed: boolean;
+    hostSourceAccessAbsent: boolean | null;
+    scriptedReplyMismatches: string[];
+    durationMs: number;
+    questions: Array<{
+        requestId: string;
+        step: number;
+        questions: Array<{
+            id: string;
+            header: string;
+            question: string;
+            options: Array<{
+                label: string;
+                description: string;
+            }> | null;
+        }>;
+        answers: Record<string, string[]>;
+    }>;
+    fallbackUserReplies?: Array<{
+        step: number;
+        eventTraceIndex: number;
+        answers: string[];
+    }>;
+    finalMessages: string[];
+    usage: unknown;
+    workspaceChanges: string[];
+    workspaceArtifacts: WorkspaceArtifactEvidence[];
+    diagnostics: string[];
+    eventTrace: unknown[];
+    isolation: CodexEvalRpcIsolationEvidence | null;
+    limitations: string[];
+    stderr?: string;
+}
+export declare const CODEX_CONVERSATION_NETWORK_LIMITATION = "Conversation app-server and tool execution network remains enabled because macOS Seatbelt cannot be nested reliably.";
+export declare const CODEX_CONVERSATION_FINAL_ANSWER_FALLBACK_LIMITATION = "For final-answer fallback turns, expected_question_count is checked against answers_by_index length because structured questions are not observable.";
+interface AppServerFrameHandler {
+    (frame: Record<string, unknown>): Promise<void> | void;
+}
+export declare function settleAppServerProcessCleanup(options: {
+    child: ChildProcess;
+    processTreeMonitor: ChildProcessTreeMonitor;
+    waitForClose: (timeoutMs: number) => Promise<boolean>;
+}): Promise<void>;
+export type CodexConversationProviderStage = 'workspace-materialization-started' | 'workspace-materialized' | 'containment-admission-started' | 'containment-admitted' | 'session-started' | 'session-initialized' | 'thread-started' | 'turn-started' | 'turn-completed';
+export declare class CodexEvalAttemptControl {
+    private readonly options;
+    private readonly controller;
+    private readonly deadlineAt?;
+    private readonly timer?;
+    private removeExternalAbort;
+    private timedOut;
+    private externallyAborted;
+    constructor(options: {
+        label: string;
+        timeoutMs?: number;
+        signal?: AbortSignal;
+    });
+    get signal(): AbortSignal;
+    get timeoutExpired(): boolean;
+    throwIfStopped(stage: string): void;
+    remaining(stage: string): number;
+    rpcTimeout(stage: string, capMs?: number): number;
+    stopError(stage: string): Error;
+    dispose(): void;
+}
+export declare function inspectAppServerConfigWarning(frame: Record<string, unknown>): string | null;
+export declare class EvalAppServerSession {
+    private readonly process;
+    private readonly processTreeId;
+    private readonly processTreeMonitor;
+    private readonly pending;
+    private readonly handlers;
+    private readonly failureHandlers;
+    private stdoutBuffer;
+    private stderrBuffer;
+    private nextId;
+    private terminalError?;
+    private closing;
+    private readonly closed;
+    private resolveClosed?;
+    private cleanupPromise?;
+    private unregisterSignalCleanup;
+    constructor(options: {
+        codexBin: string;
+        cwd: string;
+        env: NodeJS.ProcessEnv;
+        modelProvider: string;
+        sensitiveEnvironmentNames: string[];
+        isolation: AwSkillEvalIsolation;
+        processManagement?: CodexEvalProcessManagementEvidence;
+        configOverrides?: readonly string[];
+        hostReadContainmentProfile?: string;
+    });
+    onFrame(handler: AppServerFrameHandler): () => void;
+    onFailure(handler: (error: Error) => void): () => void;
+    initialize(timeoutMs?: number): Promise<void>;
+    sendRequest(method: string, params: unknown, timeoutMs?: number): Promise<unknown>;
+    respond(id: string | number, result: unknown): Promise<void>;
+    cleanup(): Promise<void>;
+    private cleanupProcess;
+    get stderr(): string;
+    scrub<T>(value: T): T;
+    private waitForClose;
+    private write;
+    private handleChunk;
+    private handleLine;
+    private fail;
+}
+export declare function normalizeConversationProviderCase(options: {
+    value: Record<string, unknown>;
+    expectedSkill: string;
+    fixtureRoot: string;
+    suitePathPrepend?: string[];
+}): ConversationEvalCase;
+export declare function extractAppServerThreadId(result: unknown): string;
+export declare function runCodexConversationProviderCase(options: {
+    evalCase: ConversationEvalCase;
+    sourceProjectRoot: string;
+    skillMode: 'with-skill' | 'without-skill' | 'previous-version';
+    skillName: string;
+    skillNames: string[];
+    codexBin: string;
+    model?: string;
+    codexRuntime: CodexEvalRuntimeConfig;
+    timeoutMs: number;
+    sourceEnv?: NodeJS.ProcessEnv;
+    isolation: AwSkillEvalIsolation;
+    forbiddenHostRoots?: string[];
+    environmentFixtures: Array<{
+        fixtureRoot: string;
+        relativePath: string;
+    }>;
+    processManagement?: CodexEvalProcessManagementEvidence;
+    workRoot?: string;
+    manageTimeout?: boolean;
+    signal?: AbortSignal;
+    onStage?: (stage: CodexConversationProviderStage) => void;
+}): Promise<AppServerRunEvidence>;
+export declare function runCodexEvalIsolationProbe(options: {
+    subjectRoot: string;
+    skillName: string;
+    skillNames?: string[];
+    codexBin: string;
+    codexRuntime: CodexEvalRuntimeConfig;
+    sourceEnv?: NodeJS.ProcessEnv;
+    processManagement?: CodexEvalProcessManagementEvidence;
+    timeoutMs?: number;
+    signal?: AbortSignal;
+}): Promise<CodexEvalRpcIsolationEvidence>;
+export {};