@lesliechan721/agent-skill-eval 0.1.0-beta.7 → 0.1.0-beta.9

package/dist/bundle.d.ts

@@ -1,93 +0,0 @@
-import type { Digest, ValidationResult } from './types.js';
-export interface EvalArtifact {
-    role: 'trace' | 'report' | 'output' | 'grading' | 'other';
-    path: string;
-    digest: Digest;
-}
-export interface EvalGrading {
-    grader: {
-        id: string;
-        version: string;
-        [key: string]: unknown;
-    };
-    result: Record<string, unknown>;
-    [key: string]: unknown;
-}
-export interface EvalRun {
-    caseId: string;
-    configuration: string;
-    comparisonId: string | null;
-    skillMode: 'with-skill' | 'without-skill' | 'previous-version';
-    skillDigest: Digest | null;
-    runIndex: number;
-    status: 'completed' | 'process-failed' | 'timeout' | 'error';
-    exitCode: number | null;
-    durationMs: number;
-    stdoutPath: string;
-    stdoutDigest: Digest;
-    stderrPath: string;
-    stderrDigest: Digest;
-    usage: Record<string, unknown> | null;
-    grading: EvalGrading | null;
-    artifacts: EvalArtifact[];
-}
-export interface ComparisonAggregate {
-    comparisonId: string;
-    caseId: string;
-    runIndex: number;
-    paired: boolean;
-    comparable: boolean;
-}
-export interface EvalRunBundle {
-    schemaVersion: 3;
-    runner: {
-        id: string;
-        version: string;
-    };
-    independentContext: boolean;
-    provider: {
-        id: string;
-        version: string;
-    };
-    planDigest: Digest;
-    targetSkillDigest: Digest;
-    caseCorpusDigest: Digest;
-    comparisonPlanned: boolean;
-    evidenceLevel: 'executed' | 'comparative';
-    evidenceRequirement?: {
-        minimumEvidenceLevel: 'executed' | 'comparative';
-        plannedComparisons: number;
-        comparableComparisons: number;
-        satisfied: boolean;
-    };
-    processManagement?: {
-        mode: 'managed' | 'verified-observed' | 'observed';
-        descendantCleanupGuaranteed: boolean;
-        limitation?: string;
-    };
-    runs: EvalRun[];
-    aggregate: {
-        cases: number;
-        configurations: number;
-        runs: number;
-        completed: number;
-        processFailed: number;
-        timedOut: number;
-        comparisons: ComparisonAggregate[];
-    };
-    limitations: string[];
-}
-export interface BundleCalculated {
-    bundleDigest: Digest;
-    cases?: number;
-    configurations?: number;
-    runs?: number;
-    completed?: number;
-    processFailed?: number;
-    timedOut?: number;
-    comparisons?: ComparisonAggregate[];
-}
-export declare function comparisonAggregate(runs: EvalRun[], gradingBoundRuns?: Set<string>): ComparisonAggregate[];
-export declare function validateRunBundle(bundlePath: string, options?: {
-    evidenceRoot?: string;
-}): Promise<ValidationResult<BundleCalculated>>;

package/dist/bundle.js

@@ -1,216 +0,0 @@
-import { lstat, readFile, realpath } from 'node:fs/promises';
-import path from 'node:path';
-import { isPathInside, sha256, stable } from './common.js';
-import { validateJsonSchema } from './json-schema.js';
-import { getSchema } from './schemas.js';
-function identity(values) {
-    return JSON.stringify(values);
-}
-function graderIdentity(run) {
-    return run.grading ? identity([run.grading.grader.id, run.grading.grader.version]) : null;
-}
-function runIdentity(run) {
-    return identity([run.caseId, run.configuration, run.runIndex]);
-}
-export function comparisonAggregate(runs, gradingBoundRuns = new Set(runs
-    .filter((run) => run.grading !== null && run.artifacts.filter((entry) => entry.role === 'grading').length === 1)
-    .map(runIdentity))) {
-    const groups = new Map();
-    for (const run of runs) {
-        if (run.comparisonId === null)
-            continue;
-        const key = identity([run.comparisonId, run.caseId, run.runIndex]);
-        const group = groups.get(key) ?? {
-            comparisonId: run.comparisonId,
-            caseId: run.caseId,
-            runIndex: run.runIndex,
-            runs: [],
-        };
-        group.runs.push(run);
-        groups.set(key, group);
-    }
-    return [...groups.values()].map((group) => {
-        const withSkill = group.runs.filter((run) => run.skillMode === 'with-skill' && run.status === 'completed');
-        const baselines = group.runs.filter((run) => run.skillMode !== 'with-skill' && run.status === 'completed');
-        const paired = withSkill.length > 0 && baselines.length > 0;
-        const comparable = paired && withSkill.some((current) => {
-            if (!gradingBoundRuns.has(runIdentity(current)))
-                return false;
-            const currentGrader = graderIdentity(current);
-            return currentGrader !== null && baselines.some((baseline) => gradingBoundRuns.has(runIdentity(baseline)) && graderIdentity(baseline) === currentGrader);
-        });
-        return { comparisonId: group.comparisonId, caseId: group.caseId, runIndex: group.runIndex, paired, comparable };
-    }).sort((left, right) => identity([left.comparisonId, left.caseId, left.runIndex])
-        .localeCompare(identity([right.comparisonId, right.caseId, right.runIndex])));
-}
-function isCanonicalEvidencePath(value) {
-    if (!value || value.includes('\\') || path.posix.isAbsolute(value) || path.win32.isAbsolute(value))
-        return false;
-    return value.split('/').every((segment) => segment && segment !== '.' && segment !== '..');
-}
-async function evidenceErrors(root, value, digest, label) {
-    if (!isCanonicalEvidencePath(value))
-        return [`${label} must be a canonical POSIX relative file path`];
-    const candidate = path.resolve(root, value);
-    if (!isPathInside(root, candidate))
-        return [`${label} leaves the evidence root`];
-    try {
-        const info = await lstat(candidate);
-        if (info.isSymbolicLink())
-            return [`${label} must not be a symlink`];
-        if (!info.isFile())
-            return [`${label} must be a regular file`];
-        const actual = await realpath(candidate);
-        if (!isPathInside(root, actual))
-            return [`${label} leaves the evidence root`];
-        return sha256(await readFile(actual)) === digest ? [] : [`${label} digest does not match file content`];
-    }
-    catch {
-        return [`${label} does not resolve to a regular evidence file`];
-    }
-}
-async function gradingBindingErrors(root, run, index) {
-    const artifacts = run.artifacts.filter((entry) => entry.role === 'grading');
-    if (run.grading === null) {
-        return artifacts.length === 0
-            ? { errors: [], bound: false }
-            : { errors: [`runs[${index}] must not have a grading artifact when grading is null`], bound: false };
-    }
-    if (artifacts.length !== 1)
-        return { errors: [`runs[${index}] grading requires exactly one grading artifact`], bound: false };
-    const artifact = artifacts[0];
-    if (!isCanonicalEvidencePath(artifact.path))
-        return { errors: [], bound: false };
-    try {
-        const candidate = path.resolve(root, artifact.path);
-        if (!isPathInside(root, candidate))
-            return { errors: [], bound: false };
-        const info = await lstat(candidate);
-        if (info.isSymbolicLink() || !info.isFile())
-            return { errors: [], bound: false };
-        const actual = await realpath(candidate);
-        if (!isPathInside(root, actual))
-            return { errors: [], bound: false };
-        const bytes = await readFile(actual);
-        if (sha256(bytes) !== artifact.digest)
-            return { errors: [], bound: false };
-        const parsed = JSON.parse(bytes.toString('utf8'));
-        if (stable(parsed) !== stable(run.grading)) {
-            return { errors: [`runs[${index}] grading artifact does not match run.grading`], bound: false };
-        }
-        return { errors: [], bound: true };
-    }
-    catch (error) {
-        if (error instanceof SyntaxError) {
-            return { errors: [`runs[${index}] grading artifact must contain JSON`], bound: false };
-        }
-        return { errors: [], bound: false };
-    }
-}
-export async function validateRunBundle(bundlePath, options = {}) {
-    let bundle;
-    const errors = [];
-    let raw;
-    try {
-        raw = await readFile(bundlePath);
-    }
-    catch (error) {
-        return { valid: false, errors: [`bundle is unavailable or invalid: ${error.message}`], calculated: null };
-    }
-    const digestCalculated = { bundleDigest: sha256(raw) };
-    try {
-        const value = JSON.parse(raw.toString('utf8'));
-        errors.push(...validateJsonSchema(await getSchema('eval-run-bundle'), value));
-        if (errors.length > 0)
-            return { valid: false, errors, calculated: digestCalculated };
-        bundle = value;
-    }
-    catch (error) {
-        return { valid: false, errors: [`bundle is unavailable or invalid: ${error.message}`], calculated: digestCalculated };
-    }
-    const root = await realpath(options.evidenceRoot ?? path.dirname(path.resolve(bundlePath)));
-    const runIdentities = new Set();
-    const gradingBoundRuns = new Set();
-    for (let index = 0; index < bundle.runs.length; index += 1) {
-        const run = bundle.runs[index];
-        const key = identity([run.caseId, run.configuration, run.runIndex]);
-        if (runIdentities.has(key))
-            errors.push(`duplicate run identity ${key}`);
-        runIdentities.add(key);
-        if (run.skillMode === 'with-skill' && run.skillDigest !== bundle.targetSkillDigest)
-            errors.push(`with-skill run ${key} must use targetSkillDigest`);
-        if (run.skillMode === 'without-skill' && run.skillDigest !== null)
-            errors.push(`without-skill run ${key} must use skillDigest=null`);
-        if (run.skillMode === 'previous-version' && run.skillDigest === null)
-            errors.push(`previous-version run ${key} must use a Skill digest`);
-        errors.push(...await evidenceErrors(root, run.stdoutPath, run.stdoutDigest, `runs[${index}].stdoutPath`), ...await evidenceErrors(root, run.stderrPath, run.stderrDigest, `runs[${index}].stderrPath`));
-        for (let artifactIndex = 0; artifactIndex < run.artifacts.length; artifactIndex += 1) {
-            const artifact = run.artifacts[artifactIndex];
-            errors.push(...await evidenceErrors(root, artifact.path, artifact.digest, `runs[${index}].artifacts[${artifactIndex}].path`));
-        }
-        const gradingBinding = await gradingBindingErrors(root, run, index);
-        errors.push(...gradingBinding.errors);
-        if (gradingBinding.bound)
-            gradingBoundRuns.add(runIdentity(run));
-    }
-    const comparisons = comparisonAggregate(bundle.runs, gradingBoundRuns);
-    const calculated = {
-        bundleDigest: digestCalculated.bundleDigest,
-        cases: new Set(bundle.runs.map((run) => run.caseId)).size,
-        configurations: new Set(bundle.runs.map((run) => run.configuration)).size,
-        runs: bundle.runs.length,
-        completed: bundle.runs.filter((run) => run.status === 'completed').length,
-        processFailed: bundle.runs.filter((run) => run.status === 'process-failed' || run.status === 'error').length,
-        timedOut: bundle.runs.filter((run) => run.status === 'timeout').length,
-        comparisons,
-    };
-    for (const name of ['cases', 'configurations', 'runs', 'completed', 'processFailed', 'timedOut'])
-        if (bundle.aggregate[name] !== calculated[name])
-            errors.push(`aggregate.${name} does not match runs`);
-    const actualComparisons = [...bundle.aggregate.comparisons].sort((left, right) => identity([left.comparisonId, left.caseId, left.runIndex]).localeCompare(identity([right.comparisonId, right.caseId, right.runIndex])));
-    if (JSON.stringify(actualComparisons) !== JSON.stringify(comparisons))
-        errors.push('aggregate.comparisons does not match runs');
-    const comparisonPlanned = comparisons.length > 0;
-    if (bundle.comparisonPlanned !== comparisonPlanned)
-        errors.push('comparisonPlanned does not match run comparison identities');
-    const evidenceLevel = comparisons.some((entry) => entry.comparable) ? 'comparative' : 'executed';
-    if (bundle.evidenceLevel !== evidenceLevel)
-        errors.push(`evidenceLevel must be ${evidenceLevel} for the supplied runs`);
-    if (bundle.evidenceRequirement) {
-        const comparableComparisons = comparisons.filter((entry) => entry.comparable).length;
-        if (bundle.evidenceRequirement.comparableComparisons !== comparableComparisons)
-            errors.push('evidenceRequirement.comparableComparisons does not match aggregate comparisons');
-        if (bundle.evidenceRequirement.plannedComparisons < comparisons.length)
-            errors.push('evidenceRequirement.plannedComparisons is smaller than aggregate comparisons');
-        const comparisonCoverageSatisfied = bundle.evidenceRequirement.minimumEvidenceLevel === 'executed' || (bundle.evidenceRequirement.plannedComparisons > 0 &&
-            comparableComparisons === bundle.evidenceRequirement.plannedComparisons);
-        if (bundle.evidenceRequirement.satisfied && !comparisonCoverageSatisfied)
-            errors.push('evidenceRequirement.satisfied cannot be true without required comparison coverage');
-    }
-    if (bundle.processManagement?.mode === 'managed' &&
-        bundle.processManagement.descendantCleanupGuaranteed !== true)
-        errors.push('managed processManagement must guarantee descendant cleanup');
-    if (bundle.processManagement &&
-        bundle.processManagement.mode !== 'managed' &&
-        bundle.processManagement.descendantCleanupGuaranteed !== false)
-        errors.push('observed processManagement must not guarantee descendant cleanup');
-    if (bundle.processManagement &&
-        bundle.processManagement.mode !== 'managed' &&
-        !bundle.processManagement.limitation)
-        errors.push('observed processManagement requires a limitation');
-    if (bundle.processManagement?.limitation &&
-        !bundle.limitations.includes(bundle.processManagement.limitation))
-        errors.push('processManagement limitation must be preserved in limitations');
-    if (bundle.evidenceLevel === 'comparative' &&
-        bundle.evidenceRequirement?.satisfied !== false &&
-        bundle.processManagement?.mode === 'observed')
-        errors.push('accepted comparative evidence requires managed or verified-observed process management');
-    if (bundle.evidenceLevel === 'comparative' &&
-        bundle.evidenceRequirement?.satisfied === true &&
-        bundle.processManagement === undefined)
-        errors.push('accepted comparative evidence requires processManagement evidence');
-    if (comparisons.some((entry) => !entry.comparable) && bundle.limitations.length === 0)
-        errors.push('non-comparable planned comparisons require a limitation');
-    return { valid: errors.length === 0, errors, calculated };
-}
-//# sourceMappingURL=bundle.js.map

package/dist/bundle.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"bundle.js","sourceRoot":"","sources":["../src/bundle.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC5D,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AAuFxC,SAAS,QAAQ,CAAC,MAAiB;IACjC,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;AAC/B,CAAC;AAED,SAAS,cAAc,CAAC,GAAY;IAClC,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;AAC3F,CAAC;AAED,SAAS,WAAW,CAAC,GAAY;IAC/B,OAAO,QAAQ,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAA;AAChE,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,IAAe,EACf,gBAAgB,GAAG,IAAI,GAAG,CACxB,IAAI;KACD,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,KAAK,IAAI,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC;KAC/G,GAAG,CAAC,WAAW,CAAC,CACpB;IAED,MAAM,MAAM,GAAG,IAAI,GAAG,EAAuF,CAAA;IAC7G,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,YAAY,KAAK,IAAI;YAAE,SAAQ;QACvC,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAA;QAClE,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI;YAC/B,YAAY,EAAE,GAAG,CAAC,YAAY;YAC9B,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,IAAI,EAAE,EAAE;SACT,CAAA;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACpB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;IACxB,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACxC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,KAAK,YAAY,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAA;QAC1G,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,KAAK,YAAY,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAA;QAC1G,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,CAAA;QAC3D,MAAM,UAAU,GAAG,MAAM,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;YACtD,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;gBAAE,OAAO,KAAK,CAAA;YAC7D,MAAM,aAAa,GAAG,cAAc,CAAC,OAAO,CAAC,CAAA;YAC7C,OAAO,aAAa,KAAK,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAC3D,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,IAAI,cAAc,CAAC,QAAQ,CAAC,KAAK,aAAa,CAAC,CAAA;QAC9F,CAAC,CAAC,CAAA;QACF,OAAO,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,CAAA;IACjH,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;SAC/E,aAAa,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;AACjF,CAAC;AAED,SAAS,uBAAuB,CAAC,KAAa;IAC5C,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IAChH,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,IAAI,OAAO,KAAK,GAAG,IAAI,OAAO,KAAK,IAAI,CAAC,CAAA;AAC5F,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,KAAa,EAAE,MAAc,EAAE,KAAa;IACtF,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,KAAK,+CAA+C,CAAC,CAAA;IACrG,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAC3C,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC;QAAE,OAAO,CAAC,GAAG,KAAK,2BAA2B,CAAC,CAAA;IAChF,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,CAAA;QACnC,IAAI,IAAI,CAAC,cAAc,EAAE;YAAE,OAAO,CAAC,GAAG,KAAK,wBAAwB,CAAC,CAAA;QACpE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAAE,OAAO,CAAC,GAAG,KAAK,yBAAyB,CAAC,CAAA;QAC9D,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAA;QACxC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC;YAAE,OAAO,CAAC,GAAG,KAAK,2BAA2B,CAAC,CAAA;QAC7E,OAAO,MAAM,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,qCAAqC,CAAC,CAAA;IACzG,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,GAAG,KAAK,8CAA8C,CAAC,CAAA;IACjE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,IAAY,EACZ,GAAY,EACZ,KAAa;IAEb,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAA;IAC3E,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QACzB,OAAO,SAAS,CAAC,MAAM,KAAK,CAAC;YAC3B,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE;YAC9B,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,QAAQ,KAAK,yDAAyD,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;IACxG,CAAC;IACD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QACxB,OAAO,EAAE,MAAM,EAAE,CAAC,QAAQ,KAAK,iDAAiD,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;IACnG,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAA;IAC7B,IAAI,CAAC,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;IAChF,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAA;QACnD,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC;YAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;QACvE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,CAAA;QACnC,IAAI,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;QAChF,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAA;QACxC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC;YAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;QACpE,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAA;QACpC,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,MAAM;YAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;QAC1E,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAY,CAAA;QAC5D,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3C,OAAO,EAAE,MAAM,EAAE,CAAC,QAAQ,KAAK,+CAA+C,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;QACjG,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAA;IACpC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,WAAW,EAAE,CAAC;YACjC,OAAO,EAAE,MAAM,EAAE,CAAC,QAAQ,KAAK,sCAAsC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;QACxF,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;IACrC,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,UAAkB,EAClB,UAAqC,EAAE;IAEvC,IAAI,MAAqB,CAAA;IACzB,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,IAAI,GAAW,CAAA;IACf,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,CAAA;IAClC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,qCAAsC,KAAe,CAAC,OAAO,EAAE,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAA;IACtH,CAAC;IACD,MAAM,gBAAgB,GAAqB,EAAE,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAA;IACxE,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAY,CAAA;QACzD,MAAM,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,MAAM,SAAS,CAAC,iBAAiB,CAAC,EAAE,KAAK,CAAC,CAAC,CAAA;QAC7E,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAA;QACpF,MAAM,GAAG,KAAsB,CAAA;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,qCAAsC,KAAe,CAAC,OAAO,EAAE,CAAC,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAA;IAClI,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;IAC3F,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAA;IACvC,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAA;IAC1C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC3D,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC9B,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAA;QACnE,IAAI,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAA;QACxE,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACtB,IAAI,GAAG,CAAC,SAAS,KAAK,YAAY,IAAI,GAAG,CAAC,WAAW,KAAK,MAAM,CAAC,iBAAiB;YAChF,MAAM,CAAC,IAAI,CAAC,kBAAkB,GAAG,6BAA6B,CAAC,CAAA;QACjE,IAAI,GAAG,CAAC,SAAS,KAAK,eAAe,IAAI,GAAG,CAAC,WAAW,KAAK,IAAI;YAC/D,MAAM,CAAC,IAAI,CAAC,qBAAqB,GAAG,4BAA4B,CAAC,CAAA;QACnE,IAAI,GAAG,CAAC,SAAS,KAAK,kBAAkB,IAAI,GAAG,CAAC,WAAW,KAAK,IAAI;YAClE,MAAM,CAAC,IAAI,CAAC,wBAAwB,GAAG,0BAA0B,CAAC,CAAA;QACpE,MAAM,CAAC,IAAI,CACT,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,YAAY,EAAE,QAAQ,KAAK,cAAc,CAAC,EAC5F,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,YAAY,EAAE,QAAQ,KAAK,cAAc,CAAC,CAC7F,CAAA;QACD,KAAK,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,aAAa,IAAI,CAAC,EAAE,CAAC;YACrF,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,aAAa,CAAC,CAAA;YAC7C,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,KAAK,eAAe,aAAa,QAAQ,CAAC,CAAC,CAAA;QAC/H,CAAC;QACD,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAA;QACnE,MAAM,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC,CAAA;QACrC,IAAI,cAAc,CAAC,KAAK;YAAE,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAA;IAClE,CAAC;IACD,MAAM,WAAW,GAAG,mBAAmB,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAA;IACtE,MAAM,UAAU,GAAqB;QACnC,YAAY,EAAE,gBAAgB,CAAC,YAAY;QAC3C,KAAK,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QACzD,cAAc,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI;QACzE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM;QACxB,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,MAAM;QACzE,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,gBAAgB,IAAI,GAAG,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM;QAC5G,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM;QACtE,WAAW;KACZ,CAAA;IACD,KAAK,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,WAAW,EAAE,eAAe,EAAE,UAAU,CAAU;QACvG,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,UAAU,CAAC,IAAI,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,aAAa,IAAI,sBAAsB,CAAC,CAAA;IACvG,MAAM,iBAAiB,GAAG,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAC/E,QAAQ,CAAC,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;IACxI,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;QACnE,MAAM,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAA;IAC1D,MAAM,iBAAiB,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAA;IAChD,IAAI,MAAM,CAAC,iBAAiB,KAAK,iBAAiB;QAChD,MAAM,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAA;IAC3E,MAAM,aAAa,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAA;IAChG,IAAI,MAAM,CAAC,aAAa,KAAK,aAAa;QACxC,MAAM,CAAC,IAAI,CAAC,yBAAyB,aAAa,wBAAwB,CAAC,CAAA;IAC7E,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAC/B,MAAM,qBAAqB,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,CAAA;QACpF,IAAI,MAAM,CAAC,mBAAmB,CAAC,qBAAqB,KAAK,qBAAqB;YAC5E,MAAM,CAAC,IAAI,CAAC,gFAAgF,CAAC,CAAA;QAC/F,IAAI,MAAM,CAAC,mBAAmB,CAAC,kBAAkB,GAAG,WAAW,CAAC,MAAM;YACpE,MAAM,CAAC,IAAI,CAAC,8EAA8E,CAAC,CAAA;QAC7F,MAAM,2BAA2B,GAAG,MAAM,CAAC,mBAAmB,CAAC,oBAAoB,KAAK,UAAU,IAAI,CACpG,MAAM,CAAC,mBAAmB,CAAC,kBAAkB,GAAG,CAAC;YACjD,qBAAqB,KAAK,MAAM,CAAC,mBAAmB,CAAC,kBAAkB,CACxE,CAAA;QACD,IAAI,MAAM,CAAC,mBAAmB,CAAC,SAAS,IAAI,CAAC,2BAA2B;YACtE,MAAM,CAAC,IAAI,CAAC,mFAAmF,CAAC,CAAA;IACpG,CAAC;IACD,IACE,MAAM,CAAC,iBAAiB,EAAE,IAAI,KAAK,SAAS;QAC5C,MAAM,CAAC,iBAAiB,CAAC,2BAA2B,KAAK,IAAI;QAC7D,MAAM,CAAC,IAAI,CAAC,6DAA6D,CAAC,CAAA;IAC5E,IACE,MAAM,CAAC,iBAAiB;QACxB,MAAM,CAAC,iBAAiB,CAAC,IAAI,KAAK,SAAS;QAC3C,MAAM,CAAC,iBAAiB,CAAC,2BAA2B,KAAK,KAAK;QAC9D,MAAM,CAAC,IAAI,CAAC,kEAAkE,CAAC,CAAA;IACjF,IACE,MAAM,CAAC,iBAAiB;QACxB,MAAM,CAAC,iBAAiB,CAAC,IAAI,KAAK,SAAS;QAC3C,CAAC,MAAM,CAAC,iBAAiB,CAAC,UAAU;QACpC,MAAM,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAA;IACjE,IACE,MAAM,CAAC,iBAAiB,EAAE,UAAU;QACpC,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,iBAAiB,CAAC,UAAU,CAAC;QACjE,MAAM,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAA;IAC9E,IACE,MAAM,CAAC,aAAa,KAAK,aAAa;QACtC,MAAM,CAAC,mBAAmB,EAAE,SAAS,KAAK,KAAK;QAC/C,MAAM,CAAC,iBAAiB,EAAE,IAAI,KAAK,UAAU;QAC7C,MAAM,CAAC,IAAI,CAAC,wFAAwF,CAAC,CAAA;IACvG,IACE,MAAM,CAAC,aAAa,KAAK,aAAa;QACtC,MAAM,CAAC,mBAAmB,EAAE,SAAS,KAAK,IAAI;QAC9C,MAAM,CAAC,iBAAiB,KAAK,SAAS;QACtC,MAAM,CAAC,IAAI,CAAC,mEAAmE,CAAC,CAAA;IAClF,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC;QACnF,MAAM,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAA;IACxE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAA;AAC3D,CAAC"}

package/dist/capabilities.d.ts

@@ -1,2 +0,0 @@
-import type { RuntimeCapabilities } from './types.js';
-export declare function getRuntimeCapabilities(platform?: NodeJS.Platform): RuntimeCapabilities;

package/dist/capabilities.js

@@ -1,38 +0,0 @@
-import { createRequire } from 'node:module';
-import process from 'node:process';
-export function getRuntimeCapabilities(platform = process.platform) {
-    const require = createRequire(import.meta.url);
-    const packageJson = require('../package.json');
-    const processObservationAvailable = platform === 'darwin' || platform === 'linux' || platform === 'win32';
-    return {
-        schemaVersion: 1,
-        provider: 'aw-skill-eval-process',
-        providerVersion: packageJson.version,
-        capabilities: {
-            triggerDiscovery: false,
-            skillPathExecution: processObservationAvailable,
-            previousVersionBaseline: processObservationAvailable,
-            withoutSkillBaseline: processObservationAvailable,
-            cleanContext: false,
-            parallelRuns: false,
-            toolTrace: false,
-            tokenUsage: true,
-            duration: true,
-            sandbox: false,
-            materialization: true,
-        },
-        limitations: [
-            'The process provider does not prove a clean model context.',
-            'The process provider does not supply a tool trace or sandbox.',
-            ...(processObservationAvailable
-                ? [
-                    `Standalone ${platform} execution uses process observation and before/after Skill digests; it does not guarantee descendant cleanup.`,
-                    ...(platform === 'darwin' || platform === 'linux'
-                        ? ['Comparative evidence requires Full Access verified process observation or confirmed external containment.']
-                        : ['Comparative evidence is disabled without confirmed external containment.']),
-                ]
-                : [`Standalone ${platform} execution is disabled because process observation is unavailable.`]),
-        ],
-    };
-}
-//# sourceMappingURL=capabilities.js.map

package/dist/capabilities.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"capabilities.js","sourceRoot":"","sources":["../src/capabilities.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAC3C,OAAO,OAAO,MAAM,cAAc,CAAA;AAGlC,MAAM,UAAU,sBAAsB,CAAC,WAA4B,OAAO,CAAC,QAAQ;IACjF,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC9C,MAAM,WAAW,GAAG,OAAO,CAAC,iBAAiB,CAAwB,CAAA;IACrE,MAAM,2BAA2B,GAAG,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,OAAO,IAAI,QAAQ,KAAK,OAAO,CAAA;IACzG,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,QAAQ,EAAE,uBAAuB;QACjC,eAAe,EAAE,WAAW,CAAC,OAAO;QACpC,YAAY,EAAE;YACZ,gBAAgB,EAAE,KAAK;YACvB,kBAAkB,EAAE,2BAA2B;YAC/C,uBAAuB,EAAE,2BAA2B;YACpD,oBAAoB,EAAE,2BAA2B;YACjD,YAAY,EAAE,KAAK;YACnB,YAAY,EAAE,KAAK;YACnB,SAAS,EAAE,KAAK;YAChB,UAAU,EAAE,IAAI;YAChB,QAAQ,EAAE,IAAI;YACd,OAAO,EAAE,KAAK;YACd,eAAe,EAAE,IAAI;SACtB;QACD,WAAW,EAAE;YACX,4DAA4D;YAC5D,+DAA+D;YAC/D,GAAG,CAAC,2BAA2B;gBAC7B,CAAC,CAAC;oBACE,cAAc,QAAQ,+GAA+G;oBACrI,GAAG,CAAC,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,OAAO;wBAC/C,CAAC,CAAC,CAAC,2GAA2G,CAAC;wBAC/G,CAAC,CAAC,CAAC,0EAA0E,CAAC,CAAC;iBAClF;gBACH,CAAC,CAAC,CAAC,cAAc,QAAQ,oEAAoE,CAAC,CAAC;SAClG;KACF,CAAA;AACH,CAAC"}

package/dist/plan.d.ts

@@ -1,79 +0,0 @@
-export { computeCaseCorpusDigest } from './corpus-digest.js';
-import { type CapturedRuntimeSkill } from './runtime-skill.js';
-import type { Digest, RuntimeCapabilities, ValidationResult } from './types.js';
-export interface EvalConfiguration {
-    id: string;
-    skillMode: 'with-skill' | 'without-skill' | 'previous-version';
-    comparisonId: string | null;
-    skillPath?: string | null;
-    skillDigest?: Digest | null;
-    env?: Record<string, string>;
-}
-export interface AwSkillEvalExecutionPlan {
-    schemaVersion: 3;
-    planId: string;
-    minimumEvidenceLevel?: 'executed' | 'comparative';
-    target: {
-        name: string;
-        path: string;
-        digest: Digest;
-    };
-    cases: {
-        paths: string[];
-        corpusDigest: Digest;
-    };
-    provider: {
-        id: string;
-        command: string[];
-        cwd?: string | null;
-        env?: Record<string, string>;
-        capabilitiesPath: string;
-        capabilitiesDigest: Digest;
-        inheritEnvironment: boolean;
-    };
-    configurations: EvalConfiguration[];
-    runsPerCase: number;
-    timeoutMs: number;
-    outputRoot: string;
-    capture: {
-        outputs: boolean;
-        trace: boolean;
-        timing: boolean;
-        usage: boolean;
-    };
-}
-export interface EvalCase extends Record<string, unknown> {
-    id?: string | number;
-}
-export interface PlanCalculated {
-    targetSkillDigest: Digest | null;
-    caseCorpusDigest: Digest | null;
-    capabilitiesDigest: Digest | null;
-    configurationSkillDigests: Record<string, Digest | null>;
-}
-export interface CapturedPlanFile {
-    label: string;
-    requestedPath: string;
-    canonicalPath: string;
-    digest: Digest;
-    raw: string;
-}
-export interface CapturedExecutionPlan {
-    plan: AwSkillEvalExecutionPlan;
-    base: string;
-    outputRoot: string;
-    planFile: CapturedPlanFile;
-    caseFiles: CapturedPlanFile[];
-    capabilitiesFile: CapturedPlanFile;
-    cases: EvalCase[];
-    capabilities: RuntimeCapabilities;
-    skills: Map<string, CapturedRuntimeSkill | null>;
-    boundSkills: Map<string, CapturedRuntimeSkill>;
-}
-export interface CapturedExecutionPlanResult extends ValidationResult<PlanCalculated> {
-    captured: CapturedExecutionPlan | null;
-}
-export declare function casesFrom(document: unknown): EvalCase[];
-export declare function promptOf(evalCase: EvalCase): string;
-export declare function captureAndValidateExecutionPlan(planPath: string): Promise<CapturedExecutionPlanResult>;
-export declare function validateExecutionPlan(planPath: string): Promise<ValidationResult<PlanCalculated>>;