@lesliechan721/agent-skill-eval 0.1.0-beta.3 → 0.1.0-beta.30

package/dist/eval-process-management.js

@@ -0,0 +1,90 @@
+import process from 'node:process';
+import { assertCodexEvalKeychainWrapperAbsent } from './codex-eval-isolation.js';
+import { assertProcessTreeInspectionAvailable, } from './process-tree.js';
+import { resolveWindowsJobHelperPath } from './windows-job.js';
+const FULL_ACCESS_PERMISSION_PROFILE = 'danger-full-access';
+const UNMANAGED_LIMITATION = 'Only the direct provider child is targeted; descendant discovery and cleanup are not available.';
+const MANAGED_LIMITATION = 'Process-tree inspection verifies cooperative descendants, but it is not OS-enforced containment and does not guarantee descendant cleanup.';
+const WINDOWS_MANAGED_LIMITATION = 'Windows uses OS-enforced Job containment; the current EvalResult contract does not claim portable descendant cleanup.';
+const UNMANAGED_RISK = 'Unmanaged mode skips process-tree inspection, targets only the direct provider child, and can leave descendant processes running. It does not bypass the macOS keychain or cmux checks.';
+export function normalizedCodexPermissionProfile(sourceEnv) {
+    const value = sourceEnv.CODEX_PERMISSION_PROFILE?.trim();
+    if (!value)
+        return null;
+    return value.replace(/^:/u, '');
+}
+export function codexPermissionProfileStatus(sourceEnv) {
+    const profile = normalizedCodexPermissionProfile(sourceEnv);
+    if (profile === null)
+        return 'unknown';
+    return profile === FULL_ACCESS_PERMISSION_PROFILE ? 'full-access' : 'restricted';
+}
+function restrictedSessionError(profile) {
+    return new Error(`Model eval is blocked because CODEX_PERMISSION_PROFILE=${profile} does not identify ` +
+        'a Full Access Codex session. A one-off sandbox_permissions=require_escalated request ' +
+        'does not change the original permission profile. Do not retry with one-off elevation. ' +
+        'Switch the entire session to Full Access or explicitly authorize ' +
+        `--allow-unmanaged-processes. ${UNMANAGED_RISK} No keychain check, process-tree inspection, output ` +
+        'directory, or model process was started.');
+}
+export async function resolveEvalProcessManagement(options) {
+    const sourceEnv = options.sourceEnv ?? process.env;
+    const dependencies = options.dependencies ?? {};
+    const platform = dependencies.platform ?? dependencies.processTree?.platform ?? process.platform;
+    const profile = normalizedCodexPermissionProfile(sourceEnv);
+    const permissionStatus = codexPermissionProfileStatus(sourceEnv);
+    if (permissionStatus === 'restricted' && !options.allowUnmanaged) {
+        throw restrictedSessionError(profile);
+    }
+    await assertCodexEvalKeychainWrapperAbsent({
+        platform,
+        sourceEnv,
+        ...(options.codexBin ? { codexBin: options.codexBin } : {}),
+        ...(dependencies.resolveCodexExecutable
+            ? { resolveCodexExecutable: dependencies.resolveCodexExecutable }
+            : {}),
+        ...(dependencies.readUserKeychainSearchList
+            ? { readUserKeychainSearchList: dependencies.readUserKeychainSearchList }
+            : {}),
+    });
+    if (options.allowUnmanaged) {
+        return {
+            mode: 'unmanaged-confirmed',
+            inspectionAvailable: false,
+            userAcknowledged: true,
+            descendantCleanupGuaranteed: false,
+            limitation: UNMANAGED_LIMITATION,
+            diagnostic: permissionStatus === 'restricted'
+                ? `Restricted Codex permission profile ${profile} was explicitly accepted; process-tree inspection was skipped.`
+                : 'Unmanaged process execution was explicitly accepted; process-tree inspection was skipped.',
+        };
+    }
+    try {
+        if (platform === 'win32') {
+            (dependencies.resolveWindowsJobHelper ?? resolveWindowsJobHelperPath)();
+        }
+        else if (dependencies.inspectProcessTree)
+            await dependencies.inspectProcessTree();
+        else {
+            await assertProcessTreeInspectionAvailable({
+                ...dependencies.processTree,
+                platform,
+            });
+        }
+        return {
+            mode: 'managed',
+            inspectionAvailable: true,
+            userAcknowledged: false,
+            descendantCleanupGuaranteed: false,
+            limitation: platform === 'win32' ? WINDOWS_MANAGED_LIMITATION : MANAGED_LIMITATION,
+        };
+    }
+    catch (error) {
+        const diagnostic = error instanceof Error ? error.message : String(error);
+        throw new Error(`Full Access process management is unavailable. ${diagnostic} No output directory ` +
+            'or model process was started. A one-off sandbox_permissions=require_escalated ' +
+            'request is not Full Access. Switch the entire session to Full Access or explicitly ' +
+            `authorize --allow-unmanaged-processes. ${UNMANAGED_RISK}`);
+    }
+}
+//# sourceMappingURL=eval-process-management.js.map

package/dist/eval-process-management.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-process-management.js","sourceRoot":"","sources":["../src/eval-process-management.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,cAAc,CAAA;AAClC,OAAO,EAAE,oCAAoC,EAAE,MAAM,2BAA2B,CAAA;AAEhF,OAAO,EACL,oCAAoC,GAErC,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAE,2BAA2B,EAAE,MAAM,kBAAkB,CAAA;AAE9D,MAAM,8BAA8B,GAAG,oBAAoB,CAAA;AAC3D,MAAM,oBAAoB,GACxB,iGAAiG,CAAA;AACnG,MAAM,kBAAkB,GACtB,4IAA4I,CAAA;AAC9I,MAAM,0BAA0B,GAC9B,uHAAuH,CAAA;AACzH,MAAM,cAAc,GAClB,yLAAyL,CAAA;AAe3L,MAAM,UAAU,gCAAgC,CAC9C,SAA4B;IAE5B,MAAM,KAAK,GAAG,SAAS,CAAC,wBAAwB,EAAE,IAAI,EAAE,CAAA;IACxD,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IACvB,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;AACjC,CAAC;AAED,MAAM,UAAU,4BAA4B,CAC1C,SAA4B;IAE5B,MAAM,OAAO,GAAG,gCAAgC,CAAC,SAAS,CAAC,CAAA;IAC3D,IAAI,OAAO,KAAK,IAAI;QAAE,OAAO,SAAS,CAAA;IACtC,OAAO,OAAO,KAAK,8BAA8B,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,CAAA;AAClF,CAAC;AAED,SAAS,sBAAsB,CAAC,OAAe;IAC7C,OAAO,IAAI,KAAK,CACd,0DAA0D,OAAO,qBAAqB;QACtF,uFAAuF;QACvF,wFAAwF;QACxF,mEAAmE;QACnE,gCAAgC,cAAc,sDAAsD;QACpG,0CAA0C,CAC3C,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAAC,OAKlD;IACC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,CAAA;IAClD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,EAAE,CAAA;IAC/C,MAAM,QAAQ,GAAG,YAAY,CAAC,QAAQ,IAAI,YAAY,CAAC,WAAW,EAAE,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAA;IAChG,MAAM,OAAO,GAAG,gCAAgC,CAAC,SAAS,CAAC,CAAA;IAC3D,MAAM,gBAAgB,GAAG,4BAA4B,CAAC,SAAS,CAAC,CAAA;IAEhE,IAAI,gBAAgB,KAAK,YAAY,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;QACjE,MAAM,sBAAsB,CAAC,OAAQ,CAAC,CAAA;IACxC,CAAC;IAED,MAAM,oCAAoC,CAAC;QACzC,QAAQ;QACR,SAAS;QACT,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,GAAG,CAAC,YAAY,CAAC,sBAAsB;YACrC,CAAC,CAAC,EAAE,sBAAsB,EAAE,YAAY,CAAC,sBAAsB,EAAE;YACjE,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,YAAY,CAAC,0BAA0B;YACzC,CAAC,CAAC,EAAE,0BAA0B,EAAE,YAAY,CAAC,0BAA0B,EAAE;YACzE,CAAC,CAAC,EAAE,CAAC;KACR,CAAC,CAAA;IAEF,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QAC3B,OAAO;YACL,IAAI,EAAE,qBAAqB;YAC3B,mBAAmB,EAAE,KAAK;YAC1B,gBAAgB,EAAE,IAAI;YACtB,2BAA2B,EAAE,KAAK;YAClC,UAAU,EAAE,oBAAoB;YAChC,UAAU,EAAE,gBAAgB,KAAK,YAAY;gBAC3C,CAAC,CAAC,uCAAuC,OAAO,gEAAgE;gBAChH,CAAC,CAAC,2FAA2F;SAChG,CAAA;IACH,CAAC;IAED,IAAI,CAAC;QACH,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;YACzB,CAAC,YAAY,CAAC,uBAAuB,IAAI,2BAA2B,CAAC,EAAE,CAAA;QACzE,CAAC;aAAM,IAAI,YAAY,CAAC,kBAAkB;YAAE,MAAM,YAAY,CAAC,kBAAkB,EAAE,CAAA;aAC9E,CAAC;YACJ,MAAM,oCAAoC,CAAC;gBACzC,GAAG,YAAY,CAAC,WAAW;gBAC3B,QAAQ;aACT,CAAC,CAAA;QACJ,CAAC;QACD,OAAO;YACL,IAAI,EAAE,SAAS;YACf,mBAAmB,EAAE,IAAI;YACzB,gBAAgB,EAAE,KAAK;YACvB,2BAA2B,EAAE,KAAK;YAClC,UAAU,EAAE,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,kBAAkB;SACnF,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,UAAU,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACzE,MAAM,IAAI,KAAK,CACb,kDAAkD,UAAU,uBAAuB;YACnF,gFAAgF;YAChF,qFAAqF;YACrF,0CAA0C,cAAc,EAAE,CAC3D,CAAA;IACH,CAAC;AACH,CAAC"}

package/dist/eval-result.d.ts

@@ -0,0 +1,79 @@
+import type { EvalProvider, EvalSpec, EvalSpecCalculated } from './eval-spec.js';
+import type { SelectedCasesDocument } from './selected-cases.js';
+import type { Digest, ValidationResult } from './types.js';
+export interface EvalGrade {
+    passed: boolean;
+    score: number | null;
+    details?: Record<string, unknown>;
+}
+export interface EvalRunError {
+    message: string;
+}
+export interface EvalResultRun {
+    caseId: string;
+    variant: 'current' | 'without-skill' | 'previous-version';
+    attempt: number;
+    status: 'completed' | 'failed' | 'timeout';
+    durationMs: number;
+    outputDir: string;
+    usage: Record<string, unknown> | null;
+    grade: EvalGrade | null;
+    error: EvalRunError | null;
+    artifacts?: Array<{
+        path: string;
+        role: string;
+    }>;
+}
+export interface EvalResultSummary {
+    passed: number;
+    failed: number;
+    timeout: number;
+    error: number;
+}
+export type EvalResultProcessManagement = {
+    mode: 'managed';
+    inspectionAvailable: true;
+    userAcknowledged: false;
+    descendantCleanupGuaranteed: false;
+    limitation: string;
+} | {
+    mode: 'unmanaged-confirmed';
+    inspectionAvailable: false;
+    userAcknowledged: true;
+    descendantCleanupGuaranteed: false;
+    limitation: string;
+    diagnostic?: string;
+};
+export interface EvalResult {
+    version: 2;
+    specDigest: Digest;
+    auditDigest: Digest;
+    targetDigest: Digest;
+    caseDigest: Digest;
+    provider: EvalProvider;
+    processManagement: EvalResultProcessManagement;
+    startedAt: string;
+    finishedAt: string;
+    runs: EvalResultRun[];
+    summary: EvalResultSummary;
+}
+export interface EvalResultBinding {
+    specDigest: Digest;
+    auditDigest: Digest;
+    targetDigest: Digest;
+    caseDigest: Digest;
+    provider: EvalProvider;
+    selectedCaseIds: readonly string[];
+    allowedVariants: readonly EvalResultRun['variant'][];
+    runs: number;
+}
+export interface EvalResultFileCalculated {
+    result: EvalResult;
+    resultDigest: Digest;
+}
+export declare function evalResultVariantsForSpec(spec: EvalSpec): EvalResultRun['variant'][];
+export declare function createEvalResultBinding(calculated: EvalSpecCalculated, auditDigest: Digest, selectedCases: Pick<SelectedCasesDocument, 'cases' | 'suite'>): EvalResultBinding;
+export declare function computeEvalResultSummary(runs: readonly EvalResultRun[]): EvalResultSummary;
+export declare function validateEvalResult(value: unknown, binding?: EvalResultBinding): Promise<ValidationResult<EvalResult>>;
+export declare function parseEvalResult(value: unknown): Promise<EvalResult>;
+export declare function validateEvalResultFile(resultPath: string, binding?: EvalResultBinding): Promise<ValidationResult<EvalResultFileCalculated>>;

package/dist/eval-result.js

@@ -0,0 +1,144 @@
+import { stable } from './common.js';
+import { readEvalContractFile } from './eval-file.js';
+import { validateJsonSchema } from './json-schema.js';
+import { getSchema } from './schemas.js';
+import { assertEvalProviderSuiteKind } from './eval-suite-binding.js';
+export function evalResultVariantsForSpec(spec) {
+    if (spec.baseline.type === 'without-skill')
+        return ['current', 'without-skill'];
+    if (spec.baseline.type === 'previous-version')
+        return ['current', 'previous-version'];
+    return ['current'];
+}
+export function createEvalResultBinding(calculated, auditDigest, selectedCases) {
+    assertEvalProviderSuiteKind(calculated.spec.provider, selectedCases.suite.kind);
+    return {
+        specDigest: calculated.specDigest,
+        auditDigest,
+        targetDigest: calculated.targetDigest,
+        caseDigest: calculated.caseDigest,
+        provider: calculated.spec.provider,
+        selectedCaseIds: selectedCases.cases.map((entry) => entry.id),
+        allowedVariants: evalResultVariantsForSpec(calculated.spec),
+        runs: calculated.spec.runs,
+    };
+}
+export function computeEvalResultSummary(runs) {
+    const summary = { passed: 0, failed: 0, timeout: 0, error: 0 };
+    for (const run of runs) {
+        if (run.status === 'timeout')
+            summary.timeout += 1;
+        else if (run.status === 'failed')
+            summary.error += 1;
+        else if (run.grade?.passed)
+            summary.passed += 1;
+        else
+            summary.failed += 1;
+    }
+    return summary;
+}
+export async function validateEvalResult(value, binding) {
+    const errors = validateJsonSchema(await getSchema('eval-result'), value);
+    if (errors.length > 0)
+        return { valid: false, errors, calculated: null };
+    const result = value;
+    const leapSecond = /:60(?=(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$)/iu;
+    const timestamp = (value) => {
+        const leap = leapSecond.test(value);
+        const parsed = Date.parse(leap ? value.replace(leapSecond, ':59') : value);
+        return leap ? parsed + 1_000 : parsed;
+    };
+    if (timestamp(result.finishedAt) < timestamp(result.startedAt))
+        errors.push('Result finishedAt must not be earlier than startedAt');
+    for (const [index, run] of result.runs.entries()) {
+        if (run.status === 'completed') {
+            if (run.grade === null)
+                errors.push(`run ${index} is completed but has no grade`);
+            if (run.error !== null)
+                errors.push(`run ${index} is completed but includes an error`);
+        }
+        else {
+            if (run.grade !== null)
+                errors.push(`run ${index} has status ${run.status} but includes a grade`);
+            if (run.error === null)
+                errors.push(`run ${index} has status ${run.status} but has no error`);
+        }
+        if (typeof result.provider === 'object'
+            && result.provider.type === 'codex-workflow'
+            && run.status === 'completed') {
+            if (run.usage === null)
+                errors.push(`run ${index} codex-workflow has no usage evidence`);
+            const expected = [
+                { path: 'workflow-evidence.json', role: 'evidence' },
+                { path: 'workflow-trace.json', role: 'trace' },
+                { path: 'model-output.txt', role: 'output' },
+                { path: 'provider-result.json', role: 'result' },
+            ];
+            if (!run.artifacts
+                || stable([...run.artifacts].sort((left, right) => left.path.localeCompare(right.path)))
+                    !== stable(expected.sort((left, right) => left.path.localeCompare(right.path))))
+                errors.push(`run ${index} codex-workflow artifacts are incomplete`);
+        }
+        else if (run.artifacts !== undefined) {
+            errors.push(`run ${index} must not include codex-workflow artifacts`);
+        }
+    }
+    const summary = computeEvalResultSummary(result.runs);
+    if (stable(summary) !== stable(result.summary))
+        errors.push('Result summary does not match runs');
+    if (binding) {
+        for (const key of ['specDigest', 'auditDigest', 'targetDigest', 'caseDigest']) {
+            if (result[key] !== binding[key])
+                errors.push(`Result ${key} does not match its binding`);
+        }
+        if (stable(result.provider) !== stable(binding.provider))
+            errors.push('Result provider does not match its binding');
+        const selectedCaseIds = new Set(binding.selectedCaseIds);
+        const allowedVariants = new Set(binding.allowedVariants);
+        const observedRuns = new Set();
+        for (const [index, run] of result.runs.entries()) {
+            if (!selectedCaseIds.has(run.caseId)) {
+                errors.push(`run ${index} caseId is not selected by its binding`);
+            }
+            if (!allowedVariants.has(run.variant)) {
+                errors.push(`run ${index} variant is not enabled by its binding`);
+            }
+            if (run.attempt > binding.runs) {
+                errors.push(`run ${index} attempt exceeds its binding`);
+            }
+            const identity = JSON.stringify([run.caseId, run.variant, run.attempt]);
+            if (observedRuns.has(identity)) {
+                errors.push(`run ${index} duplicates a case, variant, and attempt tuple`);
+            }
+            observedRuns.add(identity);
+        }
+    }
+    return { valid: errors.length === 0, errors, calculated: errors.length === 0 ? result : null };
+}
+export async function parseEvalResult(value) {
+    const validation = await validateEvalResult(value);
+    if (!validation.valid || !validation.calculated)
+        throw new Error(validation.errors.join('; '));
+    return validation.calculated;
+}
+export async function validateEvalResultFile(resultPath, binding) {
+    try {
+        const file = await readEvalContractFile(resultPath, 'EvalResult');
+        const validation = await validateEvalResult(file.value, binding);
+        if (!validation.valid || !validation.calculated)
+            return { valid: false, errors: validation.errors, calculated: null };
+        return {
+            valid: true,
+            errors: [],
+            calculated: { result: validation.calculated, resultDigest: file.digest },
+        };
+    }
+    catch (error) {
+        return {
+            valid: false,
+            errors: [`EvalResult is unavailable or invalid: ${error.message}`],
+            calculated: null,
+        };
+    }
+}
+//# sourceMappingURL=eval-result.js.map

package/dist/eval-result.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-result.js","sourceRoot":"","sources":["../src/eval-result.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACpC,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAA;AAErD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AAExC,OAAO,EAAE,2BAA2B,EAAE,MAAM,yBAAyB,CAAA;AA8ErE,MAAM,UAAU,yBAAyB,CAAC,IAAc;IACtD,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,eAAe;QAAE,OAAO,CAAC,SAAS,EAAE,eAAe,CAAC,CAAA;IAC/E,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,kBAAkB;QAAE,OAAO,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAA;IACrF,OAAO,CAAC,SAAS,CAAC,CAAA;AACpB,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,UAA8B,EAC9B,WAAmB,EACnB,aAA6D;IAE7D,2BAA2B,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC/E,OAAO;QACL,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,WAAW;QACX,YAAY,EAAE,UAAU,CAAC,YAAY;QACrC,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,QAAQ,EAAE,UAAU,CAAC,IAAI,CAAC,QAAQ;QAClC,eAAe,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7D,eAAe,EAAE,yBAAyB,CAAC,UAAU,CAAC,IAAI,CAAC;QAC3D,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI;KAC3B,CAAA;AACH,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,IAA8B;IACrE,MAAM,OAAO,GAAsB,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;IACjF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO,CAAC,OAAO,IAAI,CAAC,CAAA;aAC7C,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ;YAAE,OAAO,CAAC,KAAK,IAAI,CAAC,CAAA;aAC/C,IAAI,GAAG,CAAC,KAAK,EAAE,MAAM;YAAE,OAAO,CAAC,MAAM,IAAI,CAAC,CAAA;;YAC1C,OAAO,CAAC,MAAM,IAAI,CAAC,CAAA;IAC1B,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,KAAc,EACd,OAA2B;IAE3B,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,SAAS,CAAC,aAAa,CAAC,EAAE,KAAK,CAAC,CAAA;IACxE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAA;IACxE,MAAM,MAAM,GAAG,KAAmB,CAAA;IAClC,MAAM,UAAU,GAAG,2CAA2C,CAAA;IAC9D,MAAM,SAAS,GAAG,CAAC,KAAa,EAAU,EAAE;QAC1C,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;QAC1E,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,MAAM,CAAA;IACvC,CAAC,CAAA;IACD,IAAI,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC;QAC5D,MAAM,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAA;IACrE,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QACjD,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAC/B,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,gCAAgC,CAAC,CAAA;YACjF,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,qCAAqC,CAAC,CAAA;QACxF,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,eAAe,GAAG,CAAC,MAAM,uBAAuB,CAAC,CAAA;YACjG,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,eAAe,GAAG,CAAC,MAAM,mBAAmB,CAAC,CAAA;QAC/F,CAAC;QACD,IACE,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ;eAChC,MAAM,CAAC,QAAQ,CAAC,IAAI,KAAK,gBAAgB;eACzC,GAAG,CAAC,MAAM,KAAK,WAAW,EAC7B,CAAC;YACD,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,uCAAuC,CAAC,CAAA;YACxF,MAAM,QAAQ,GAAG;gBACf,EAAE,IAAI,EAAE,wBAAwB,EAAE,IAAI,EAAE,UAAU,EAAE;gBACpD,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,OAAO,EAAE;gBAC9C,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC5C,EAAE,IAAI,EAAE,sBAAsB,EAAE,IAAI,EAAE,QAAQ,EAAE;aACjD,CAAA;YACD,IACE,CAAC,GAAG,CAAC,SAAS;mBACX,MAAM,CAAC,CAAC,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;wBAClF,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;gBACjF,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,0CAA0C,CAAC,CAAA;QACvE,CAAC;aAAM,IAAI,GAAG,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,4CAA4C,CAAC,CAAA;QACvE,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,wBAAwB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACrD,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC;QAAE,MAAM,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAA;IACjG,IAAI,OAAO,EAAE,CAAC;QACZ,KAAK,MAAM,GAAG,IAAI,CAAC,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,YAAY,CAAU,EAAE,CAAC;YACvF,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,GAAG,CAAC;gBAAE,MAAM,CAAC,IAAI,CAAC,UAAU,GAAG,6BAA6B,CAAC,CAAA;QAC3F,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC;YACtD,MAAM,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAA;QAC3D,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;QACxD,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;QACxD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAA;QACtC,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACjD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBACrC,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,wCAAwC,CAAC,CAAA;YACnE,CAAC;YACD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBACtC,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,wCAAwC,CAAC,CAAA;YACnE,CAAC;YACD,IAAI,GAAG,CAAC,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC/B,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,8BAA8B,CAAC,CAAA;YACzD,CAAC;YACD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAA;YACvE,IAAI,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC/B,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,gDAAgD,CAAC,CAAA;YAC3E,CAAC;YACD,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;AAChG,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,KAAc;IAClD,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAA;IAClD,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,UAAU,CAAC,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;IAC9F,OAAO,UAAU,CAAC,UAAU,CAAA;AAC9B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,UAAkB,EAClB,OAA2B;IAE3B,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,oBAAoB,CAAC,UAAU,EAAE,YAAY,CAAC,CAAA;QACjE,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;QAChE,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,UAAU,CAAC,UAAU;YAC7C,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAA;QACtE,OAAO;YACL,KAAK,EAAE,IAAI;YACX,MAAM,EAAE,EAAE;YACV,UAAU,EAAE,EAAE,MAAM,EAAE,UAAU,CAAC,UAAU,EAAE,YAAY,EAAE,IAAI,CAAC,MAAM,EAAE;SACzE,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,CAAC,yCAA0C,KAAe,CAAC,OAAO,EAAE,CAAC;YAC7E,UAAU,EAAE,IAAI;SACjB,CAAA;IACH,CAAC;AACH,CAAC"}

package/dist/eval-run.d.ts

@@ -0,0 +1,27 @@
+import { type CustomProcessExecutorOptions, type EvalProviderRunResult } from './custom-process.js';
+import { type EvalResult, type EvalResultProcessManagement } from './eval-result.js';
+import { type EvalProcessManagementDependencies } from './eval-process-management.js';
+import { type BuiltinEvalProviderDependencies, type BuiltinEvalProviderRunInput } from './providers.js';
+import { type CodexWorkflowExecutorOptions } from './codex-workflow.js';
+import { type SelectedCasesDocument, type SelectedEvalCase } from './selected-cases.js';
+import type { Digest } from './types.js';
+export interface RunEvalSpecOptions {
+    auditPath: string;
+    allowUnmanagedProcesses?: boolean;
+    processManagementDependencies?: EvalProcessManagementDependencies;
+    onProcessManagementResolved?: (evidence: EvalResultProcessManagement) => void;
+    signal?: AbortSignal;
+    sourceEnv?: NodeJS.ProcessEnv;
+    builtinProviderDependencies?: BuiltinEvalProviderDependencies;
+    runBuiltinProvider?: (id: string, input: BuiltinEvalProviderRunInput, dependencies?: BuiltinEvalProviderDependencies) => Promise<EvalProviderRunResult>;
+    executeCustomProvider?: (options: CustomProcessExecutorOptions) => Promise<EvalProviderRunResult>;
+    executeCodexWorkflowProvider?: (options: CodexWorkflowExecutorOptions) => Promise<EvalProviderRunResult>;
+}
+export interface RunEvalSpecResult {
+    valid: true;
+    resultPath: string;
+    resultDigest: Digest;
+    result: EvalResult;
+}
+export declare function adaptConversationProviderCase(document: SelectedCasesDocument, evalCase: SelectedEvalCase): Record<string, unknown>;
+export declare function runEvalSpec(specPath: string, options: RunEvalSpecOptions): Promise<RunEvalSpecResult>;