@lesliechan721/agent-skill-eval 0.1.0-beta.1

package/dist/bin.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/bin.js

@@ -0,0 +1,80 @@
+#!/usr/bin/env node
+import path from 'node:path';
+import process from 'node:process';
+import { computeSkillDigest, getRuntimeCapabilities, getSchema, runExecutionPlan, validateExecutionPlan, validateRunBundle, } from './index.js';
+import { isSchemaName } from './schemas.js';
+const help = `Usage: aw-skill-eval <command>
+
+Commands:
+  capabilities --json
+  schema print <aw-skill-eval-execution-plan|eval-run-bundle>
+  plan validate <plan.json>
+  run <plan.json>
+  bundle validate <bundle.json>
+  digest skill <skill-dir>
+`;
+function requirePath(value, usage) {
+    if (!value)
+        throw new Error(usage);
+    return path.resolve(value);
+}
+async function main(argv) {
+    if (argv.length === 0 || argv[0] === '--help' || argv[0] === '-h') {
+        process.stdout.write(help);
+        return;
+    }
+    const [command, action, value, ...extra] = argv;
+    if (extra.length > 0)
+        throw new Error('Too many arguments. Use --help for usage.');
+    if (command === 'capabilities') {
+        if (action !== '--json' || value !== undefined)
+            throw new Error('Usage: aw-skill-eval capabilities --json');
+        process.stdout.write(`${JSON.stringify(getRuntimeCapabilities(), null, 2)}\n`);
+        return;
+    }
+    if (command === 'schema') {
+        if (action !== 'print' || !value || !isSchemaName(value) || value === 'eval-runtime-capabilities')
+            throw new Error('Usage: aw-skill-eval schema print <aw-skill-eval-execution-plan|eval-run-bundle>');
+        process.stdout.write(`${JSON.stringify(await getSchema(value), null, 2)}\n`);
+        return;
+    }
+    if (command === 'plan') {
+        if (action !== 'validate')
+            throw new Error('Usage: aw-skill-eval plan validate <plan.json>');
+        const result = await validateExecutionPlan(requirePath(value, 'Usage: aw-skill-eval plan validate <plan.json>'));
+        process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+        if (!result.valid)
+            process.exitCode = 1;
+        return;
+    }
+    if (command === 'run') {
+        if (value !== undefined)
+            throw new Error('Usage: aw-skill-eval run <plan.json>');
+        const result = await runExecutionPlan(requirePath(action, 'Usage: aw-skill-eval run <plan.json>'));
+        process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+        if (!result.requirementsMet)
+            process.exitCode = 1;
+        return;
+    }
+    if (command === 'bundle') {
+        if (action !== 'validate')
+            throw new Error('Usage: aw-skill-eval bundle validate <bundle.json>');
+        const result = await validateRunBundle(requirePath(value, 'Usage: aw-skill-eval bundle validate <bundle.json>'));
+        process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+        if (!result.valid)
+            process.exitCode = 1;
+        return;
+    }
+    if (command === 'digest') {
+        if (action !== 'skill')
+            throw new Error('Usage: aw-skill-eval digest skill <skill-dir>');
+        process.stdout.write(`${await computeSkillDigest(requirePath(value, 'Usage: aw-skill-eval digest skill <skill-dir>'))}\n`);
+        return;
+    }
+    throw new Error(`Unknown command: ${command}. Use --help for usage.`);
+}
+main(process.argv.slice(2)).catch((error) => {
+    process.stderr.write(`[ERROR] ${error.message}\n`);
+    process.exitCode = 1;
+});
+//# sourceMappingURL=bin.js.map

package/dist/bin.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bin.js","sourceRoot":"","sources":["../src/bin.ts"],"names":[],"mappings":";AACA,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,OAAO,MAAM,cAAc,CAAA;AAClC,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACtB,SAAS,EACT,gBAAgB,EAChB,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAE3C,MAAM,IAAI,GAAG;;;;;;;;;CASZ,CAAA;AAED,SAAS,WAAW,CAAC,KAAyB,EAAE,KAAa;IAC3D,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,CAAA;IAClC,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;AAC5B,CAAC;AAED,KAAK,UAAU,IAAI,CAAC,IAAc;IAChC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAClE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAC1B,OAAM;IACR,CAAC;IACD,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC,GAAG,IAAI,CAAA;IAC/C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;IAClF,IAAI,OAAO,KAAK,cAAc,EAAE,CAAC;QAC/B,IAAI,MAAM,KAAK,QAAQ,IAAI,KAAK,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAA;QAC3G,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,sBAAsB,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;QAC9E,OAAM;IACR,CAAC;IACD,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,IAAI,MAAM,KAAK,OAAO,IAAI,CAAC,KAAK,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,2BAA2B;YAC/F,MAAM,IAAI,KAAK,CAAC,kFAAkF,CAAC,CAAA;QACrG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,SAAS,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;QAC5E,OAAM;IACR,CAAC;IACD,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;QACvB,IAAI,MAAM,KAAK,UAAU;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAA;QAC5F,MAAM,MAAM,GAAG,MAAM,qBAAqB,CAAC,WAAW,CAAC,KAAK,EAAE,gDAAgD,CAAC,CAAC,CAAA;QAChH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,MAAM,CAAC,KAAK;YAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;QACvC,OAAM;IACR,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,EAAE,CAAC;QACtB,IAAI,KAAK,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;QAChF,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,MAAM,EAAE,sCAAsC,CAAC,CAAC,CAAA;QAClG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,MAAM,CAAC,eAAe;YAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;QACjD,OAAM;IACR,CAAC;IACD,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,IAAI,MAAM,KAAK,UAAU;YAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAA;QAChG,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,WAAW,CAAC,KAAK,EAAE,oDAAoD,CAAC,CAAC,CAAA;QAChH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,MAAM,CAAC,KAAK;YAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;QACvC,OAAM;IACR,CAAC;IACD,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,IAAI,MAAM,KAAK,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAA;QACxF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,kBAAkB,CAAC,WAAW,CAAC,KAAK,EAAE,+CAA+C,CAAC,CAAC,IAAI,CAAC,CAAA;QAC1H,OAAM;IACR,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,yBAAyB,CAAC,CAAA;AACvE,CAAC;AAED,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IACnD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAY,KAAe,CAAC,OAAO,IAAI,CAAC,CAAA;IAC7D,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;AACtB,CAAC,CAAC,CAAA"}

package/dist/bundle.d.ts

@@ -0,0 +1,93 @@
+import type { Digest, ValidationResult } from './types.js';
+export interface EvalArtifact {
+    role: 'trace' | 'report' | 'output' | 'grading' | 'other';
+    path: string;
+    digest: Digest;
+}
+export interface EvalGrading {
+    grader: {
+        id: string;
+        version: string;
+        [key: string]: unknown;
+    };
+    result: Record<string, unknown>;
+    [key: string]: unknown;
+}
+export interface EvalRun {
+    caseId: string;
+    configuration: string;
+    comparisonId: string | null;
+    skillMode: 'with-skill' | 'without-skill' | 'previous-version';
+    skillDigest: Digest | null;
+    runIndex: number;
+    status: 'completed' | 'process-failed' | 'timeout' | 'error';
+    exitCode: number | null;
+    durationMs: number;
+    stdoutPath: string;
+    stdoutDigest: Digest;
+    stderrPath: string;
+    stderrDigest: Digest;
+    usage: Record<string, unknown> | null;
+    grading: EvalGrading | null;
+    artifacts: EvalArtifact[];
+}
+export interface ComparisonAggregate {
+    comparisonId: string;
+    caseId: string;
+    runIndex: number;
+    paired: boolean;
+    comparable: boolean;
+}
+export interface EvalRunBundle {
+    schemaVersion: 3;
+    runner: {
+        id: string;
+        version: string;
+    };
+    independentContext: boolean;
+    provider: {
+        id: string;
+        version: string;
+    };
+    planDigest: Digest;
+    targetSkillDigest: Digest;
+    caseCorpusDigest: Digest;
+    comparisonPlanned: boolean;
+    evidenceLevel: 'executed' | 'comparative';
+    evidenceRequirement?: {
+        minimumEvidenceLevel: 'executed' | 'comparative';
+        plannedComparisons: number;
+        comparableComparisons: number;
+        satisfied: boolean;
+    };
+    processManagement?: {
+        mode: 'managed' | 'verified-observed' | 'observed';
+        descendantCleanupGuaranteed: boolean;
+        limitation?: string;
+    };
+    runs: EvalRun[];
+    aggregate: {
+        cases: number;
+        configurations: number;
+        runs: number;
+        completed: number;
+        processFailed: number;
+        timedOut: number;
+        comparisons: ComparisonAggregate[];
+    };
+    limitations: string[];
+}
+export interface BundleCalculated {
+    bundleDigest: Digest;
+    cases?: number;
+    configurations?: number;
+    runs?: number;
+    completed?: number;
+    processFailed?: number;
+    timedOut?: number;
+    comparisons?: ComparisonAggregate[];
+}
+export declare function comparisonAggregate(runs: EvalRun[], gradingBoundRuns?: Set<string>): ComparisonAggregate[];
+export declare function validateRunBundle(bundlePath: string, options?: {
+    evidenceRoot?: string;
+}): Promise<ValidationResult<BundleCalculated>>;

package/dist/bundle.js

@@ -0,0 +1,216 @@
+import { lstat, readFile, realpath } from 'node:fs/promises';
+import path from 'node:path';
+import { isPathInside, sha256, stable } from './common.js';
+import { validateJsonSchema } from './json-schema.js';
+import { getSchema } from './schemas.js';
+function identity(values) {
+    return JSON.stringify(values);
+}
+function graderIdentity(run) {
+    return run.grading ? identity([run.grading.grader.id, run.grading.grader.version]) : null;
+}
+function runIdentity(run) {
+    return identity([run.caseId, run.configuration, run.runIndex]);
+}
+export function comparisonAggregate(runs, gradingBoundRuns = new Set(runs
+    .filter((run) => run.grading !== null && run.artifacts.filter((entry) => entry.role === 'grading').length === 1)
+    .map(runIdentity))) {
+    const groups = new Map();
+    for (const run of runs) {
+        if (run.comparisonId === null)
+            continue;
+        const key = identity([run.comparisonId, run.caseId, run.runIndex]);
+        const group = groups.get(key) ?? {
+            comparisonId: run.comparisonId,
+            caseId: run.caseId,
+            runIndex: run.runIndex,
+            runs: [],
+        };
+        group.runs.push(run);
+        groups.set(key, group);
+    }
+    return [...groups.values()].map((group) => {
+        const withSkill = group.runs.filter((run) => run.skillMode === 'with-skill' && run.status === 'completed');
+        const baselines = group.runs.filter((run) => run.skillMode !== 'with-skill' && run.status === 'completed');
+        const paired = withSkill.length > 0 && baselines.length > 0;
+        const comparable = paired && withSkill.some((current) => {
+            if (!gradingBoundRuns.has(runIdentity(current)))
+                return false;
+            const currentGrader = graderIdentity(current);
+            return currentGrader !== null && baselines.some((baseline) => gradingBoundRuns.has(runIdentity(baseline)) && graderIdentity(baseline) === currentGrader);
+        });
+        return { comparisonId: group.comparisonId, caseId: group.caseId, runIndex: group.runIndex, paired, comparable };
+    }).sort((left, right) => identity([left.comparisonId, left.caseId, left.runIndex])
+        .localeCompare(identity([right.comparisonId, right.caseId, right.runIndex])));
+}
+function isCanonicalEvidencePath(value) {
+    if (!value || value.includes('\\') || path.posix.isAbsolute(value) || path.win32.isAbsolute(value))
+        return false;
+    return value.split('/').every((segment) => segment && segment !== '.' && segment !== '..');
+}
+async function evidenceErrors(root, value, digest, label) {
+    if (!isCanonicalEvidencePath(value))
+        return [`${label} must be a canonical POSIX relative file path`];
+    const candidate = path.resolve(root, value);
+    if (!isPathInside(root, candidate))
+        return [`${label} leaves the evidence root`];
+    try {
+        const info = await lstat(candidate);
+        if (info.isSymbolicLink())
+            return [`${label} must not be a symlink`];
+        if (!info.isFile())
+            return [`${label} must be a regular file`];
+        const actual = await realpath(candidate);
+        if (!isPathInside(root, actual))
+            return [`${label} leaves the evidence root`];
+        return sha256(await readFile(actual)) === digest ? [] : [`${label} digest does not match file content`];
+    }
+    catch {
+        return [`${label} does not resolve to a regular evidence file`];
+    }
+}
+async function gradingBindingErrors(root, run, index) {
+    const artifacts = run.artifacts.filter((entry) => entry.role === 'grading');
+    if (run.grading === null) {
+        return artifacts.length === 0
+            ? { errors: [], bound: false }
+            : { errors: [`runs[${index}] must not have a grading artifact when grading is null`], bound: false };
+    }
+    if (artifacts.length !== 1)
+        return { errors: [`runs[${index}] grading requires exactly one grading artifact`], bound: false };
+    const artifact = artifacts[0];
+    if (!isCanonicalEvidencePath(artifact.path))
+        return { errors: [], bound: false };
+    try {
+        const candidate = path.resolve(root, artifact.path);
+        if (!isPathInside(root, candidate))
+            return { errors: [], bound: false };
+        const info = await lstat(candidate);
+        if (info.isSymbolicLink() || !info.isFile())
+            return { errors: [], bound: false };
+        const actual = await realpath(candidate);
+        if (!isPathInside(root, actual))
+            return { errors: [], bound: false };
+        const bytes = await readFile(actual);
+        if (sha256(bytes) !== artifact.digest)
+            return { errors: [], bound: false };
+        const parsed = JSON.parse(bytes.toString('utf8'));
+        if (stable(parsed) !== stable(run.grading)) {
+            return { errors: [`runs[${index}] grading artifact does not match run.grading`], bound: false };
+        }
+        return { errors: [], bound: true };
+    }
+    catch (error) {
+        if (error instanceof SyntaxError) {
+            return { errors: [`runs[${index}] grading artifact must contain JSON`], bound: false };
+        }
+        return { errors: [], bound: false };
+    }
+}
+export async function validateRunBundle(bundlePath, options = {}) {
+    let bundle;
+    const errors = [];
+    let raw;
+    try {
+        raw = await readFile(bundlePath);
+    }
+    catch (error) {
+        return { valid: false, errors: [`bundle is unavailable or invalid: ${error.message}`], calculated: null };
+    }
+    const digestCalculated = { bundleDigest: sha256(raw) };
+    try {
+        const value = JSON.parse(raw.toString('utf8'));
+        errors.push(...validateJsonSchema(await getSchema('eval-run-bundle'), value));
+        if (errors.length > 0)
+            return { valid: false, errors, calculated: digestCalculated };
+        bundle = value;
+    }
+    catch (error) {
+        return { valid: false, errors: [`bundle is unavailable or invalid: ${error.message}`], calculated: digestCalculated };
+    }
+    const root = await realpath(options.evidenceRoot ?? path.dirname(path.resolve(bundlePath)));
+    const runIdentities = new Set();
+    const gradingBoundRuns = new Set();
+    for (let index = 0; index < bundle.runs.length; index += 1) {
+        const run = bundle.runs[index];
+        const key = identity([run.caseId, run.configuration, run.runIndex]);
+        if (runIdentities.has(key))
+            errors.push(`duplicate run identity ${key}`);
+        runIdentities.add(key);
+        if (run.skillMode === 'with-skill' && run.skillDigest !== bundle.targetSkillDigest)
+            errors.push(`with-skill run ${key} must use targetSkillDigest`);
+        if (run.skillMode === 'without-skill' && run.skillDigest !== null)
+            errors.push(`without-skill run ${key} must use skillDigest=null`);
+        if (run.skillMode === 'previous-version' && run.skillDigest === null)
+            errors.push(`previous-version run ${key} must use a Skill digest`);
+        errors.push(...await evidenceErrors(root, run.stdoutPath, run.stdoutDigest, `runs[${index}].stdoutPath`), ...await evidenceErrors(root, run.stderrPath, run.stderrDigest, `runs[${index}].stderrPath`));
+        for (let artifactIndex = 0; artifactIndex < run.artifacts.length; artifactIndex += 1) {
+            const artifact = run.artifacts[artifactIndex];
+            errors.push(...await evidenceErrors(root, artifact.path, artifact.digest, `runs[${index}].artifacts[${artifactIndex}].path`));
+        }
+        const gradingBinding = await gradingBindingErrors(root, run, index);
+        errors.push(...gradingBinding.errors);
+        if (gradingBinding.bound)
+            gradingBoundRuns.add(runIdentity(run));
+    }
+    const comparisons = comparisonAggregate(bundle.runs, gradingBoundRuns);
+    const calculated = {
+        bundleDigest: digestCalculated.bundleDigest,
+        cases: new Set(bundle.runs.map((run) => run.caseId)).size,
+        configurations: new Set(bundle.runs.map((run) => run.configuration)).size,
+        runs: bundle.runs.length,
+        completed: bundle.runs.filter((run) => run.status === 'completed').length,
+        processFailed: bundle.runs.filter((run) => run.status === 'process-failed' || run.status === 'error').length,
+        timedOut: bundle.runs.filter((run) => run.status === 'timeout').length,
+        comparisons,
+    };
+    for (const name of ['cases', 'configurations', 'runs', 'completed', 'processFailed', 'timedOut'])
+        if (bundle.aggregate[name] !== calculated[name])
+            errors.push(`aggregate.${name} does not match runs`);
+    const actualComparisons = [...bundle.aggregate.comparisons].sort((left, right) => identity([left.comparisonId, left.caseId, left.runIndex]).localeCompare(identity([right.comparisonId, right.caseId, right.runIndex])));
+    if (JSON.stringify(actualComparisons) !== JSON.stringify(comparisons))
+        errors.push('aggregate.comparisons does not match runs');
+    const comparisonPlanned = comparisons.length > 0;
+    if (bundle.comparisonPlanned !== comparisonPlanned)
+        errors.push('comparisonPlanned does not match run comparison identities');
+    const evidenceLevel = comparisons.some((entry) => entry.comparable) ? 'comparative' : 'executed';
+    if (bundle.evidenceLevel !== evidenceLevel)
+        errors.push(`evidenceLevel must be ${evidenceLevel} for the supplied runs`);
+    if (bundle.evidenceRequirement) {
+        const comparableComparisons = comparisons.filter((entry) => entry.comparable).length;
+        if (bundle.evidenceRequirement.comparableComparisons !== comparableComparisons)
+            errors.push('evidenceRequirement.comparableComparisons does not match aggregate comparisons');
+        if (bundle.evidenceRequirement.plannedComparisons < comparisons.length)
+            errors.push('evidenceRequirement.plannedComparisons is smaller than aggregate comparisons');
+        const comparisonCoverageSatisfied = bundle.evidenceRequirement.minimumEvidenceLevel === 'executed' || (bundle.evidenceRequirement.plannedComparisons > 0 &&
+            comparableComparisons === bundle.evidenceRequirement.plannedComparisons);
+        if (bundle.evidenceRequirement.satisfied && !comparisonCoverageSatisfied)
+            errors.push('evidenceRequirement.satisfied cannot be true without required comparison coverage');
+    }
+    if (bundle.processManagement?.mode === 'managed' &&
+        bundle.processManagement.descendantCleanupGuaranteed !== true)
+        errors.push('managed processManagement must guarantee descendant cleanup');
+    if (bundle.processManagement &&
+        bundle.processManagement.mode !== 'managed' &&
+        bundle.processManagement.descendantCleanupGuaranteed !== false)
+        errors.push('observed processManagement must not guarantee descendant cleanup');
+    if (bundle.processManagement &&
+        bundle.processManagement.mode !== 'managed' &&
+        !bundle.processManagement.limitation)
+        errors.push('observed processManagement requires a limitation');
+    if (bundle.processManagement?.limitation &&
+        !bundle.limitations.includes(bundle.processManagement.limitation))
+        errors.push('processManagement limitation must be preserved in limitations');
+    if (bundle.evidenceLevel === 'comparative' &&
+        bundle.evidenceRequirement?.satisfied !== false &&
+        bundle.processManagement?.mode === 'observed')
+        errors.push('accepted comparative evidence requires managed or verified-observed process management');
+    if (bundle.evidenceLevel === 'comparative' &&
+        bundle.evidenceRequirement?.satisfied === true &&
+        bundle.processManagement === undefined)
+        errors.push('accepted comparative evidence requires processManagement evidence');
+    if (comparisons.some((entry) => !entry.comparable) && bundle.limitations.length === 0)
+        errors.push('non-comparable planned comparisons require a limitation');
+    return { valid: errors.length === 0, errors, calculated };
+}
+//# sourceMappingURL=bundle.js.map

package/dist/bundle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bundle.js","sourceRoot":"","sources":["../src/bundle.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC5D,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AAuFxC,SAAS,QAAQ,CAAC,MAAiB;IACjC,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;AAC/B,CAAC;AAED,SAAS,cAAc,CAAC,GAAY;IAClC,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;AAC3F,CAAC;AAED,SAAS,WAAW,CAAC,GAAY;IAC/B,OAAO,QAAQ,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAA;AAChE,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,IAAe,EACf,gBAAgB,GAAG,IAAI,GAAG,CACxB,IAAI;KACD,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,KAAK,IAAI,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC;KAC/G,GAAG,CAAC,WAAW,CAAC,CACpB;IAED,MAAM,MAAM,GAAG,IAAI,GAAG,EAAuF,CAAA;IAC7G,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,YAAY,KAAK,IAAI;YAAE,SAAQ;QACvC,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAA;QAClE,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI;YAC/B,YAAY,EAAE,GAAG,CAAC,YAAY;YAC9B,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,IAAI,EAAE,EAAE;SACT,CAAA;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACpB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;IACxB,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACxC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,KAAK,YAAY,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAA;QAC1G,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,KAAK,YAAY,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAA;QAC1G,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,CAAA;QAC3D,MAAM,UAAU,GAAG,MAAM,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;YACtD,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;gBAAE,OAAO,KAAK,CAAA;YAC7D,MAAM,aAAa,GAAG,cAAc,CAAC,OAAO,CAAC,CAAA;YAC7C,OAAO,aAAa,KAAK,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAC3D,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,IAAI,cAAc,CAAC,QAAQ,CAAC,KAAK,aAAa,CAAC,CAAA;QAC9F,CAAC,CAAC,CAAA;QACF,OAAO,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,CAAA;IACjH,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;SAC/E,aAAa,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;AACjF,CAAC;AAED,SAAS,uBAAuB,CAAC,KAAa;IAC5C,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IAChH,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,IAAI,OAAO,KAAK,GAAG,IAAI,OAAO,KAAK,IAAI,CAAC,CAAA;AAC5F,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,KAAa,EAAE,MAAc,EAAE,KAAa;IACtF,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,KAAK,+CAA+C,CAAC,CAAA;IACrG,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAC3C,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC;QAAE,OAAO,CAAC,GAAG,KAAK,2BAA2B,CAAC,CAAA;IAChF,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,CAAA;QACnC,IAAI,IAAI,CAAC,cAAc,EAAE;YAAE,OAAO,CAAC,GAAG,KAAK,wBAAwB,CAAC,CAAA;QACpE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAAE,OAAO,CAAC,GAAG,KAAK,yBAAyB,CAAC,CAAA;QAC9D,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAA;QACxC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC;YAAE,OAAO,CAAC,GAAG,KAAK,2BAA2B,CAAC,CAAA;QAC7E,OAAO,MAAM,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,qCAAqC,CAAC,CAAA;IACzG,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,GAAG,KAAK,8CAA8C,CAAC,CAAA;IACjE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,IAAY,EACZ,GAAY,EACZ,KAAa;IAEb,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAA;IAC3E,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QACzB,OAAO,SAAS,CAAC,MAAM,KAAK,CAAC;YAC3B,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE;YAC9B,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,QAAQ,KAAK,yDAAyD,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;IACxG,CAAC;IACD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QACxB,OAAO,EAAE,MAAM,EAAE,CAAC,QAAQ,KAAK,iDAAiD,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;IACnG,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAA;IAC7B,IAAI,CAAC,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;IAChF,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAA;QACnD,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC;YAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;QACvE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,CAAA;QACnC,IAAI,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;QAChF,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAA;QACxC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC;YAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;QACpE,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAA;QACpC,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,MAAM;YAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;QAC1E,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAY,CAAA;QAC5D,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3C,OAAO,EAAE,MAAM,EAAE,CAAC,QAAQ,KAAK,+CAA+C,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;QACjG,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAA;IACpC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,WAAW,EAAE,CAAC;YACjC,OAAO,EAAE,MAAM,EAAE,CAAC,QAAQ,KAAK,sCAAsC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;QACxF,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;IACrC,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,UAAkB,EAClB,UAAqC,EAAE;IAEvC,IAAI,MAAqB,CAAA;IACzB,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,IAAI,GAAW,CAAA;IACf,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,CAAA;IAClC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,qCAAsC,KAAe,CAAC,OAAO,EAAE,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAA;IACtH,CAAC;IACD,MAAM,gBAAgB,GAAqB,EAAE,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAA;IACxE,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAY,CAAA;QACzD,MAAM,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,MAAM,SAAS,CAAC,iBAAiB,CAAC,EAAE,KAAK,CAAC,CAAC,CAAA;QAC7E,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAA;QACpF,MAAM,GAAG,KAAsB,CAAA;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,qCAAsC,KAAe,CAAC,OAAO,EAAE,CAAC,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAA;IAClI,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;IAC3F,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAA;IACvC,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAA;IAC1C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC3D,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC9B,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAA;QACnE,IAAI,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAA;QACxE,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACtB,IAAI,GAAG,CAAC,SAAS,KAAK,YAAY,IAAI,GAAG,CAAC,WAAW,KAAK,MAAM,CAAC,iBAAiB;YAChF,MAAM,CAAC,IAAI,CAAC,kBAAkB,GAAG,6BAA6B,CAAC,CAAA;QACjE,IAAI,GAAG,CAAC,SAAS,KAAK,eAAe,IAAI,GAAG,CAAC,WAAW,KAAK,IAAI;YAC/D,MAAM,CAAC,IAAI,CAAC,qBAAqB,GAAG,4BAA4B,CAAC,CAAA;QACnE,IAAI,GAAG,CAAC,SAAS,KAAK,kBAAkB,IAAI,GAAG,CAAC,WAAW,KAAK,IAAI;YAClE,MAAM,CAAC,IAAI,CAAC,wBAAwB,GAAG,0BAA0B,CAAC,CAAA;QACpE,MAAM,CAAC,IAAI,CACT,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,YAAY,EAAE,QAAQ,KAAK,cAAc,CAAC,EAC5F,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,YAAY,EAAE,QAAQ,KAAK,cAAc,CAAC,CAC7F,CAAA;QACD,KAAK,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,aAAa,IAAI,CAAC,EAAE,CAAC;YACrF,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,aAAa,CAAC,CAAA;YAC7C,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,KAAK,eAAe,aAAa,QAAQ,CAAC,CAAC,CAAA;QAC/H,CAAC;QACD,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAA;QACnE,MAAM,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC,CAAA;QACrC,IAAI,cAAc,CAAC,KAAK;YAAE,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAA;IAClE,CAAC;IACD,MAAM,WAAW,GAAG,mBAAmB,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAA;IACtE,MAAM,UAAU,GAAqB;QACnC,YAAY,EAAE,gBAAgB,CAAC,YAAY;QAC3C,KAAK,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QACzD,cAAc,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI;QACzE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM;QACxB,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,MAAM;QACzE,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,gBAAgB,IAAI,GAAG,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM;QAC5G,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM;QACtE,WAAW;KACZ,CAAA;IACD,KAAK,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,WAAW,EAAE,eAAe,EAAE,UAAU,CAAU;QACvG,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,UAAU,CAAC,IAAI,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,aAAa,IAAI,sBAAsB,CAAC,CAAA;IACvG,MAAM,iBAAiB,GAAG,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAC/E,QAAQ,CAAC,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;IACxI,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;QACnE,MAAM,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAA;IAC1D,MAAM,iBAAiB,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAA;IAChD,IAAI,MAAM,CAAC,iBAAiB,KAAK,iBAAiB;QAChD,MAAM,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAA;IAC3E,MAAM,aAAa,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAA;IAChG,IAAI,MAAM,CAAC,aAAa,KAAK,aAAa;QACxC,MAAM,CAAC,IAAI,CAAC,yBAAyB,aAAa,wBAAwB,CAAC,CAAA;IAC7E,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAC/B,MAAM,qBAAqB,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,CAAA;QACpF,IAAI,MAAM,CAAC,mBAAmB,CAAC,qBAAqB,KAAK,qBAAqB;YAC5E,MAAM,CAAC,IAAI,CAAC,gFAAgF,CAAC,CAAA;QAC/F,IAAI,MAAM,CAAC,mBAAmB,CAAC,kBAAkB,GAAG,WAAW,CAAC,MAAM;YACpE,MAAM,CAAC,IAAI,CAAC,8EAA8E,CAAC,CAAA;QAC7F,MAAM,2BAA2B,GAAG,MAAM,CAAC,mBAAmB,CAAC,oBAAoB,KAAK,UAAU,IAAI,CACpG,MAAM,CAAC,mBAAmB,CAAC,kBAAkB,GAAG,CAAC;YACjD,qBAAqB,KAAK,MAAM,CAAC,mBAAmB,CAAC,kBAAkB,CACxE,CAAA;QACD,IAAI,MAAM,CAAC,mBAAmB,CAAC,SAAS,IAAI,CAAC,2BAA2B;YACtE,MAAM,CAAC,IAAI,CAAC,mFAAmF,CAAC,CAAA;IACpG,CAAC;IACD,IACE,MAAM,CAAC,iBAAiB,EAAE,IAAI,KAAK,SAAS;QAC5C,MAAM,CAAC,iBAAiB,CAAC,2BAA2B,KAAK,IAAI;QAC7D,MAAM,CAAC,IAAI,CAAC,6DAA6D,CAAC,CAAA;IAC5E,IACE,MAAM,CAAC,iBAAiB;QACxB,MAAM,CAAC,iBAAiB,CAAC,IAAI,KAAK,SAAS;QAC3C,MAAM,CAAC,iBAAiB,CAAC,2BAA2B,KAAK,KAAK;QAC9D,MAAM,CAAC,IAAI,CAAC,kEAAkE,CAAC,CAAA;IACjF,IACE,MAAM,CAAC,iBAAiB;QACxB,MAAM,CAAC,iBAAiB,CAAC,IAAI,KAAK,SAAS;QAC3C,CAAC,MAAM,CAAC,iBAAiB,CAAC,UAAU;QACpC,MAAM,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAA;IACjE,IACE,MAAM,CAAC,iBAAiB,EAAE,UAAU;QACpC,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,iBAAiB,CAAC,UAAU,CAAC;QACjE,MAAM,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAA;IAC9E,IACE,MAAM,CAAC,aAAa,KAAK,aAAa;QACtC,MAAM,CAAC,mBAAmB,EAAE,SAAS,KAAK,KAAK;QAC/C,MAAM,CAAC,iBAAiB,EAAE,IAAI,KAAK,UAAU;QAC7C,MAAM,CAAC,IAAI,CAAC,wFAAwF,CAAC,CAAA;IACvG,IACE,MAAM,CAAC,aAAa,KAAK,aAAa;QACtC,MAAM,CAAC,mBAAmB,EAAE,SAAS,KAAK,IAAI;QAC9C,MAAM,CAAC,iBAAiB,KAAK,SAAS;QACtC,MAAM,CAAC,IAAI,CAAC,mEAAmE,CAAC,CAAA;IAClF,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC;QACnF,MAAM,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAA;IACxE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAA;AAC3D,CAAC"}

package/dist/capabilities.d.ts

@@ -0,0 +1,2 @@
+import type { RuntimeCapabilities } from './types.js';
+export declare function getRuntimeCapabilities(platform?: NodeJS.Platform): RuntimeCapabilities;

package/dist/capabilities.js

@@ -0,0 +1,38 @@
+import { createRequire } from 'node:module';
+import process from 'node:process';
+export function getRuntimeCapabilities(platform = process.platform) {
+    const require = createRequire(import.meta.url);
+    const packageJson = require('../package.json');
+    const processObservationAvailable = platform === 'darwin' || platform === 'linux' || platform === 'win32';
+    return {
+        schemaVersion: 1,
+        provider: 'aw-skill-eval-process',
+        providerVersion: packageJson.version,
+        capabilities: {
+            triggerDiscovery: false,
+            skillPathExecution: processObservationAvailable,
+            previousVersionBaseline: processObservationAvailable,
+            withoutSkillBaseline: processObservationAvailable,
+            cleanContext: false,
+            parallelRuns: false,
+            toolTrace: false,
+            tokenUsage: true,
+            duration: true,
+            sandbox: false,
+            materialization: true,
+        },
+        limitations: [
+            'The process provider does not prove a clean model context.',
+            'The process provider does not supply a tool trace or sandbox.',
+            ...(processObservationAvailable
+                ? [
+                    `Standalone ${platform} execution uses process observation and before/after Skill digests; it does not guarantee descendant cleanup.`,
+                    ...(platform === 'darwin' || platform === 'linux'
+                        ? ['Comparative evidence requires Full Access verified process observation or confirmed external containment.']
+                        : ['Comparative evidence is disabled without confirmed external containment.']),
+                ]
+                : [`Standalone ${platform} execution is disabled because process observation is unavailable.`]),
+        ],
+    };
+}
+//# sourceMappingURL=capabilities.js.map

package/dist/capabilities.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"capabilities.js","sourceRoot":"","sources":["../src/capabilities.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAC3C,OAAO,OAAO,MAAM,cAAc,CAAA;AAGlC,MAAM,UAAU,sBAAsB,CAAC,WAA4B,OAAO,CAAC,QAAQ;IACjF,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC9C,MAAM,WAAW,GAAG,OAAO,CAAC,iBAAiB,CAAwB,CAAA;IACrE,MAAM,2BAA2B,GAAG,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,OAAO,IAAI,QAAQ,KAAK,OAAO,CAAA;IACzG,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,QAAQ,EAAE,uBAAuB;QACjC,eAAe,EAAE,WAAW,CAAC,OAAO;QACpC,YAAY,EAAE;YACZ,gBAAgB,EAAE,KAAK;YACvB,kBAAkB,EAAE,2BAA2B;YAC/C,uBAAuB,EAAE,2BAA2B;YACpD,oBAAoB,EAAE,2BAA2B;YACjD,YAAY,EAAE,KAAK;YACnB,YAAY,EAAE,KAAK;YACnB,SAAS,EAAE,KAAK;YAChB,UAAU,EAAE,IAAI;YAChB,QAAQ,EAAE,IAAI;YACd,OAAO,EAAE,KAAK;YACd,eAAe,EAAE,IAAI;SACtB;QACD,WAAW,EAAE;YACX,4DAA4D;YAC5D,+DAA+D;YAC/D,GAAG,CAAC,2BAA2B;gBAC7B,CAAC,CAAC;oBACE,cAAc,QAAQ,+GAA+G;oBACrI,GAAG,CAAC,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,OAAO;wBAC/C,CAAC,CAAC,CAAC,2GAA2G,CAAC;wBAC/G,CAAC,CAAC,CAAC,0EAA0E,CAAC,CAAC;iBAClF;gBACH,CAAC,CAAC,CAAC,cAAc,QAAQ,oEAAoE,CAAC,CAAC;SAClG;KACF,CAAA;AACH,CAAC"}

package/dist/common.d.ts

@@ -0,0 +1,8 @@
+import { canonicalJson } from './corpus-digest.js';
+import type { Digest } from './types.js';
+export declare function sha256(data: string | NodeJS.ArrayBufferView): Digest;
+export declare function isPathInside(root: string, candidate: string): boolean;
+export declare function pathsOverlap(left: string, right: string): boolean;
+export declare function safeSegment(value: string): boolean;
+export declare const stable: typeof canonicalJson;
+export declare function toPosixRelative(root: string, candidate: string): string;

package/dist/common.js

@@ -0,0 +1,22 @@
+import { createHash } from 'node:crypto';
+import path from 'node:path';
+import { canonicalJson } from './corpus-digest.js';
+export function sha256(data) {
+    return `sha256:${createHash('sha256').update(data).digest('hex')}`;
+}
+export function isPathInside(root, candidate) {
+    const relative = path.relative(root, candidate);
+    return (relative === '' ||
+        (!path.isAbsolute(relative) && relative !== '..' && !relative.startsWith(`..${path.sep}`)));
+}
+export function pathsOverlap(left, right) {
+    return isPathInside(left, right) || isPathInside(right, left);
+}
+export function safeSegment(value) {
+    return /^[A-Za-z0-9][A-Za-z0-9._-]*$/u.test(value);
+}
+export const stable = canonicalJson;
+export function toPosixRelative(root, candidate) {
+    return path.relative(root, candidate).split(path.sep).join('/');
+}
+//# sourceMappingURL=common.js.map

package/dist/common.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"common.js","sourceRoot":"","sources":["../src/common.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAGlD,MAAM,UAAU,MAAM,CAAC,IAAqC;IAC1D,OAAO,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;AACpE,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,SAAiB;IAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;IAC/C,OAAO,CACL,QAAQ,KAAK,EAAE;QACf,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,QAAQ,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAC3F,CAAA;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,KAAa;IACtD,OAAO,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;AAC/D,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,OAAO,+BAA+B,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;AACpD,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG,aAAa,CAAA;AAEnC,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,SAAiB;IAC7D,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AACjE,CAAC"}

package/dist/corpus-digest.d.ts

@@ -0,0 +1,3 @@
+import type { Digest } from './types.js';
+export declare function canonicalJson(value: unknown): string;
+export declare function computeCaseCorpusDigest(casePaths: string[]): Promise<Digest>;

package/dist/corpus-digest.js

@@ -0,0 +1,19 @@
+import { createHash } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+export function canonicalJson(value) {
+    if (Array.isArray(value))
+        return `[${value.map(canonicalJson).join(',')}]`;
+    if (value && typeof value === 'object') {
+        return `{${Object.keys(value)
+            .sort()
+            .map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`)
+            .join(',')}}`;
+    }
+    return JSON.stringify(value);
+}
+export async function computeCaseCorpusDigest(casePaths) {
+    const documents = await Promise.all(casePaths.map(async (filePath) => canonicalJson(JSON.parse(await readFile(filePath, 'utf8')))));
+    documents.sort();
+    return `sha256:${createHash('sha256').update(documents.join('\n')).digest('hex')}`;
+}
+//# sourceMappingURL=corpus-digest.js.map

package/dist/corpus-digest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"corpus-digest.js","sourceRoot":"","sources":["../src/corpus-digest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAG3C,MAAM,UAAU,aAAa,CAAC,KAAc;IAC1C,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAA;IAC1E,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;aAC1B,IAAI,EAAE;aACN,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,aAAa,CAAE,KAAiC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;aAChG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAA;IACjB,CAAC;IACD,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;AAC9B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,SAAmB;IAC/D,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CACjC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAC/F,CAAA;IACD,SAAS,CAAC,IAAI,EAAE,CAAA;IAChB,OAAO,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;AACpF,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,16 @@
+export { validateExecutionPlan } from './plan.js';
+export { canonicalJson, computeCaseCorpusDigest } from './corpus-digest.js';
+export { getRuntimeCapabilities } from './capabilities.js';
+export { runExecutionPlan } from './run.js';
+export { validateRunBundle } from './bundle.js';
+export { computeSkillDigest, createRuntimeSkillSnapshot } from './runtime-skill.js';
+export type { RuntimeSkillDigestOptions } from './runtime-skill.js';
+export { assertProviderProcessSupported, ProviderProcessCleanupError, runProviderProcess, } from './process.js';
+export type { ProviderProcessDependencies } from './process.js';
+export { assertProcessTreeInspectionAvailable, observeProcessTree, terminateProcessTree, } from './process-tree.js';
+export { getSchema, getSchemaDigest, validateRuntimeCapabilities } from './schemas.js';
+export { validateJsonSchema } from './json-schema.js';
+export type { AwSkillEvalExecutionPlan, EvalCase, EvalConfiguration, PlanCalculated, } from './plan.js';
+export type { BundleCalculated, ComparisonAggregate, EvalArtifact, EvalGrading, EvalRun, EvalRunBundle, } from './bundle.js';
+export type { Digest, ExternalContainmentConfirmation, ProcessLifecycleHooks, ProcessRequest, ProcessResult, ProcessTreeCleanupConfirmation, ProcessTerminationReason, ProviderProcessHandle, RunExecutionPlanOptions, RunExecutionResult, RuntimeCapabilities, RuntimeSkillSnapshot, SchemaName, ValidationResult, } from './types.js';
+export type { ProcessTreeDependencies, ProcessTreeObservation, ProcessTreeTerminationOptions, } from './process-tree.js';

package/dist/index.js

@@ -0,0 +1,11 @@
+export { validateExecutionPlan } from './plan.js';
+export { canonicalJson, computeCaseCorpusDigest } from './corpus-digest.js';
+export { getRuntimeCapabilities } from './capabilities.js';
+export { runExecutionPlan } from './run.js';
+export { validateRunBundle } from './bundle.js';
+export { computeSkillDigest, createRuntimeSkillSnapshot } from './runtime-skill.js';
+export { assertProviderProcessSupported, ProviderProcessCleanupError, runProviderProcess, } from './process.js';
+export { assertProcessTreeInspectionAvailable, observeProcessTree, terminateProcessTree, } from './process-tree.js';
+export { getSchema, getSchemaDigest, validateRuntimeCapabilities } from './schemas.js';
+export { validateJsonSchema } from './json-schema.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAA;AACjD,OAAO,EAAE,aAAa,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAA;AAC3E,OAAO,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAA;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAA;AAC3C,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAC/C,OAAO,EAAE,kBAAkB,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAA;AAEnF,OAAO,EACL,8BAA8B,EAC9B,2BAA2B,EAC3B,kBAAkB,GACnB,MAAM,cAAc,CAAA;AAErB,OAAO,EACL,oCAAoC,EACpC,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,2BAA2B,EAAE,MAAM,cAAc,CAAA;AACtF,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA"}

package/dist/json-schema.d.ts

@@ -0,0 +1,3 @@
+type JsonSchema = Record<string, unknown>;
+export declare function validateJsonSchema(schema: JsonSchema, value: unknown): string[];
+export {};