@lesliechan721/agent-skill-eval 0.1.0-beta.7 → 0.1.0-beta.9

package/dist/schemas.d.ts

@@ -1,5 +1,3 @@
-import type { Digest, RuntimeCapabilities, SchemaName, ValidationResult } from './types.js';
-export declare function isSchemaName(value: string): value is SchemaName;
+import type { Digest, SchemaName } from './types.js';
 export declare function getSchema(name: SchemaName): Promise<Record<string, unknown>>;
 export declare function getSchemaDigest(name: SchemaName): Promise<Digest>;
-export declare function validateRuntimeCapabilities(value: unknown): Promise<ValidationResult<RuntimeCapabilities>>;

package/dist/schemas.js

@@ -1,13 +1,4 @@
 import { readFile } from 'node:fs/promises';
-import { validateJsonSchema } from './json-schema.js';
-const schemaNames = new Set([
-    'aw-skill-eval-execution-plan',
-    'eval-run-bundle',
-    'eval-runtime-capabilities',
-]);
-export function isSchemaName(value) {
-    return schemaNames.has(value);
-}
 export async function getSchema(name) {
     const source = new URL(`./schemas/${name}.schema.json`, import.meta.url);
     return JSON.parse(await readFile(source, 'utf8'));
@@ -16,12 +7,4 @@ export async function getSchemaDigest(name) {
     const digests = JSON.parse(await readFile(new URL('./schemas/digests.json', import.meta.url), 'utf8'));
     return digests[name];
 }
-export async function validateRuntimeCapabilities(value) {
-    const errors = validateJsonSchema(await getSchema('eval-runtime-capabilities'), value);
-    return {
-        valid: errors.length === 0,
-        errors,
-        calculated: errors.length === 0 ? value : null,
-    };
-}
 //# sourceMappingURL=schemas.js.map

package/dist/schemas.js.map

@@ -1 +1 @@
-{"version":3,"file":"schemas.js","sourceRoot":"","sources":["../src/schemas.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAQrD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAa;IACtC,8BAA8B;IAC9B,iBAAiB;IACjB,2BAA2B;CAC5B,CAAC,CAAA;AAEF,MAAM,UAAU,YAAY,CAAC,KAAa;IACxC,OAAO,WAAW,CAAC,GAAG,CAAC,KAAmB,CAAC,CAAA;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAgB;IAC9C,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,aAAa,IAAI,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACxE,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAA4B,CAAA;AAC9E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAgB;IACpD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CACxB,MAAM,QAAQ,CAAC,IAAI,GAAG,CAAC,wBAAwB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAC7C,CAAA;IAC/B,OAAO,OAAO,CAAC,IAAI,CAAC,CAAA;AACtB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,KAAc;IAEd,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,SAAS,CAAC,2BAA2B,CAAC,EAAE,KAAK,CAAC,CAAA;IACtF,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;QAC1B,MAAM;QACN,UAAU,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAA4B,CAAC,CAAC,CAAC,IAAI;KACtE,CAAA;AACH,CAAC"}
+{"version":3,"file":"schemas.js","sourceRoot":"","sources":["../src/schemas.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAM3C,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAgB;IAC9C,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,aAAa,IAAI,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACxE,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAA4B,CAAA;AAC9E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAgB;IACpD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CACxB,MAAM,QAAQ,CAAC,IAAI,GAAG,CAAC,wBAAwB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAC7C,CAAA;IAC/B,OAAO,OAAO,CAAC,IAAI,CAAC,CAAA;AACtB,CAAC"}

package/dist/selected-cases.d.ts

@@ -0,0 +1,30 @@
+import { type AwSkillEvalTriggerCase, type AwSkillEvalTriggerSuiteCaseContract } from './codex-eval-cases.js';
+import type { ValidationResult } from './types.js';
+export type SelectedCaseSuiteKind = 'trigger' | 'conversation' | 'custom-process' | 'codex-workflow';
+export interface SelectedCaseSuite {
+    kind: SelectedCaseSuiteKind;
+    unitUnderTest: string;
+    participants?: string[];
+    policy?: {
+        optimization?: string;
+        falseNegativeCost?: string;
+        falsePositiveCost?: string;
+        rationale?: string;
+    };
+    runtime?: Record<string, unknown>;
+    readonly providerSuite?: AwSkillEvalTriggerSuiteCaseContract;
+}
+export interface SelectedEvalCase extends Record<string, unknown> {
+    id: string;
+    split: string;
+    tags: string[];
+    readonly providerCase?: AwSkillEvalTriggerCase;
+}
+export interface SelectedCasesDocument {
+    version: 1;
+    suite: SelectedCaseSuite;
+    cases: SelectedEvalCase[];
+}
+export declare function parseSelectedCases(value: unknown): SelectedCasesDocument;
+export declare function validateSelectedCases(value: unknown): Promise<ValidationResult<SelectedCasesDocument>>;
+export declare function validateSelectedCasesFile(casesPath: string): Promise<ValidationResult<SelectedCasesDocument>>;

package/dist/selected-cases.js

@@ -0,0 +1,388 @@
+import { readEvalContractFile } from './eval-file.js';
+import { normalizeAwSkillEvalTriggerCase, } from './codex-eval-cases.js';
+function record(value, label) {
+    if (!value || typeof value !== 'object' || Array.isArray(value)) {
+        throw new Error(`${label} must be an object.`);
+    }
+    return value;
+}
+function onlyKeys(value, allowed, label) {
+    const keys = new Set(allowed);
+    const unknown = Object.keys(value).find((key) => !keys.has(key));
+    if (unknown)
+        throw new Error(`${label} has unknown field ${JSON.stringify(unknown)}.`);
+}
+function nonEmptyString(value, label) {
+    if (typeof value !== 'string' || value.trim() === '') {
+        throw new Error(`${label} must be a non-empty string.`);
+    }
+    return value;
+}
+function safeId(value, label) {
+    const result = nonEmptyString(value, label);
+    if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/u.test(result)) {
+        throw new Error(`${label} must be a safe path segment.`);
+    }
+    return result;
+}
+function uniqueStrings(value, label, allowEmpty = false) {
+    if (!Array.isArray(value) || (!allowEmpty && value.length === 0)) {
+        throw new Error(`${label} must be ${allowEmpty ? 'an' : 'a non-empty'} array.`);
+    }
+    const result = value.map((entry, index) => nonEmptyString(entry, `${label}[${index}]`));
+    if (new Set(result).size !== result.length)
+        throw new Error(`${label} must contain unique values.`);
+    return result;
+}
+function assertJsonValue(value, label) {
+    if (value === null || typeof value === 'string' || typeof value === 'boolean')
+        return;
+    if (typeof value === 'number' && Number.isFinite(value))
+        return;
+    if (Array.isArray(value)) {
+        value.forEach((entry, index) => assertJsonValue(entry, `${label}[${index}]`));
+        return;
+    }
+    const row = record(value, label);
+    for (const [key, entry] of Object.entries(row)) {
+        if (key === '__proto__' || key === 'prototype' || key === 'constructor') {
+            throw new Error(`${label} contains unsafe field ${JSON.stringify(key)}.`);
+        }
+        assertJsonValue(entry, `${label}.${key}`);
+    }
+}
+function optionalJson(value, label) {
+    if (value !== undefined)
+        assertJsonValue(value, label);
+    return value;
+}
+function relativePath(value, label, options = {}) {
+    const raw = nonEmptyString(value, label);
+    const treePattern = options.allowTreePattern === true && raw.endsWith('/**');
+    const candidate = treePattern ? raw.slice(0, -3) : raw;
+    if (candidate === ''
+        || candidate.includes('\0')
+        || candidate.includes('\\')
+        || candidate.includes('*')
+        || candidate.includes('?')
+        || candidate.includes('[')
+        || candidate.includes(']')
+        || candidate.startsWith('/')
+        || /^[A-Za-z]:/u.test(candidate)
+        || candidate.split('/').some((segment) => segment === '' || segment === '.' || segment === '..'))
+        throw new Error(`${label} must be a normalized relative POSIX path.`);
+    return raw;
+}
+function pathArray(value, label, options = {}) {
+    const values = uniqueStrings(value, label, options.allowEmpty === true);
+    return values.map((entry, index) => relativePath(entry, `${label}[${index}]`, { allowTreePattern: options.allowTreePattern }));
+}
+function triggerSuiteContract(suite) {
+    const runtime = suite.runtime ?? {};
+    return {
+        id: 'selected',
+        kind: 'trigger',
+        skill: suite.unitUnderTest,
+        skills: [...new Set([suite.unitUnderTest, ...(suite.participants ?? [])])],
+        cases: 'cases.json',
+        observeSpawnAgent: runtime.observeSpawnAgent === true,
+        isolation: runtime.isolation === 'strict' ? 'strict' : 'standard',
+        strictNegativeSplits: (runtime.strictNegativeSplits ?? []),
+        forbidMultipleSkillLoads: runtime.forbidMultipleSkillLoads === true,
+        ...(typeof runtime.fixture === 'string' ? { fixtureModule: runtime.fixture } : {}),
+        profiles: ['selected'],
+        defaultRuns: 1,
+        releaseSplits: ['all'],
+        requiredForRelease: false,
+        perRunTimeoutMs: 1,
+    };
+}
+function parseRuntime(value, kind) {
+    const runtime = record(value, 'selected cases.suite.runtime');
+    if (kind === 'trigger') {
+        onlyKeys(runtime, [
+            'observeSpawnAgent',
+            'isolation',
+            'strictNegativeSplits',
+            'forbidMultipleSkillLoads',
+            'fixture',
+            'minimumPassRate',
+        ], 'selected cases.suite.runtime');
+        if (runtime.observeSpawnAgent !== undefined && typeof runtime.observeSpawnAgent !== 'boolean') {
+            throw new Error('selected cases.suite.runtime.observeSpawnAgent must be boolean.');
+        }
+        if (runtime.forbidMultipleSkillLoads !== undefined && typeof runtime.forbidMultipleSkillLoads !== 'boolean') {
+            throw new Error('selected cases.suite.runtime.forbidMultipleSkillLoads must be boolean.');
+        }
+        if (runtime.isolation !== undefined && runtime.isolation !== 'standard' && runtime.isolation !== 'strict') {
+            throw new Error('selected cases.suite.runtime.isolation must be standard or strict.');
+        }
+        if (runtime.strictNegativeSplits !== undefined) {
+            const splits = uniqueStrings(runtime.strictNegativeSplits, 'selected cases.suite.runtime.strictNegativeSplits', true);
+            const invalid = splits.find((split) => split !== 'train' && split !== 'validation' && split !== 'sanity');
+            if (invalid) {
+                throw new Error('selected cases.suite.runtime.strictNegativeSplits has an invalid split.');
+            }
+        }
+        if (runtime.fixture !== undefined) {
+            runtime.fixture = relativePath(runtime.fixture, 'selected cases.suite.runtime.fixture');
+        }
+    }
+    else if (kind === 'conversation') {
+        onlyKeys(runtime, ['isolation', 'pathPrepend', 'minimumPassRate'], 'selected cases.suite.runtime');
+        if (runtime.isolation !== undefined && runtime.isolation !== 'standard' && runtime.isolation !== 'strict') {
+            throw new Error('selected cases.suite.runtime.isolation must be standard or strict.');
+        }
+        runtime.pathPrepend = runtime.pathPrepend === undefined
+            ? []
+            : pathArray(runtime.pathPrepend, 'selected cases.suite.runtime.pathPrepend', { allowEmpty: true });
+    }
+    else {
+        onlyKeys(runtime, ['minimumPassRate'], 'selected cases.suite.runtime');
+    }
+    if (runtime.minimumPassRate !== undefined
+        && (typeof runtime.minimumPassRate !== 'number'
+            || !Number.isFinite(runtime.minimumPassRate)
+            || runtime.minimumPassRate < 0
+            || runtime.minimumPassRate > 1)) {
+        throw new Error('selected cases.suite.runtime.minimumPassRate must be a finite number from 0 to 1.');
+    }
+    return runtime;
+}
+function parseSuite(value) {
+    const suite = record(value, 'selected cases.suite');
+    onlyKeys(suite, ['kind', 'unitUnderTest', 'participants', 'policy', 'runtime'], 'selected cases.suite');
+    if (suite.kind !== 'trigger'
+        && suite.kind !== 'conversation'
+        && suite.kind !== 'custom-process'
+        && suite.kind !== 'codex-workflow') {
+        throw new Error('selected cases.suite.kind must be trigger, conversation, custom-process, or codex-workflow.');
+    }
+    const result = {
+        kind: suite.kind,
+        unitUnderTest: safeId(suite.unitUnderTest, 'selected cases.suite.unitUnderTest'),
+    };
+    if (suite.participants !== undefined) {
+        result.participants = uniqueStrings(suite.participants, 'selected cases.suite.participants')
+            .map((entry, index) => safeId(entry, `selected cases.suite.participants[${index}]`));
+    }
+    if (suite.policy !== undefined) {
+        const policy = record(suite.policy, 'selected cases.suite.policy');
+        onlyKeys(policy, ['optimization', 'falseNegativeCost', 'falsePositiveCost', 'rationale'], 'selected cases.suite.policy');
+        result.policy = Object.fromEntries(Object.entries(policy).map(([key, entry]) => [
+            key,
+            nonEmptyString(entry, `selected cases.suite.policy.${key}`),
+        ]));
+    }
+    if (suite.runtime !== undefined) {
+        result.runtime = parseRuntime(suite.runtime, suite.kind);
+    }
+    if (result.kind === 'trigger') {
+        Object.defineProperty(result, 'providerSuite', {
+            value: triggerSuiteContract(result),
+            enumerable: false,
+        });
+    }
+    return result;
+}
+function triggerExpectation(row, label) {
+    if (row.expectedLoad !== undefined && row.expected !== undefined) {
+        throw new Error(`${label} must not contain both expectedLoad and expected.`);
+    }
+    if (row.expectedLoad !== undefined) {
+        if (typeof row.expectedLoad !== 'boolean')
+            throw new Error(`${label}.expectedLoad must be boolean.`);
+        return { should_load_skill: row.expectedLoad };
+    }
+    const expected = record(row.expected, `${label}.expected`);
+    onlyKeys(expected, ['loadedSkills'], `${label}.expected`);
+    const loadedSkills = uniqueStrings(expected.loadedSkills, `${label}.expected.loadedSkills`, true);
+    return { expected_loaded_skills: loadedSkills };
+}
+function parseTriggerCase(row, index, suite) {
+    const label = `selected cases.cases[${index}]`;
+    onlyKeys(row, ['id', 'split', 'tags', 'input', 'audit'], label);
+    const input = record(row.input, `${label}.input`);
+    onlyKeys(input, ['query', 'expectedLoad', 'expected', 'shouldSpawnAgent', 'workspaceFixture', 'useProjectRules'], `${label}.input`);
+    const id = safeId(row.id, `${label}.id`);
+    const split = safeId(row.split, `${label}.split`);
+    const tags = uniqueStrings(row.tags, `${label}.tags`, true);
+    const canonical = normalizeAwSkillEvalTriggerCase({
+        id,
+        split,
+        query: nonEmptyString(input.query, `${label}.input.query`),
+        ...triggerExpectation(input, `${label}.input`),
+        ...(input.shouldSpawnAgent === undefined
+            ? {}
+            : { should_spawn_agent: input.shouldSpawnAgent }),
+        ...(input.workspaceFixture === undefined
+            ? {}
+            : { workspace_fixture: relativePath(input.workspaceFixture, `${label}.input.workspaceFixture`) }),
+        ...(input.useProjectRules === undefined
+            ? {}
+            : { use_project_rules: input.useProjectRules }),
+        ...(tags.length === 0 ? {} : { tags }),
+    }, index, suite.providerSuite);
+    if (canonical.workspace_fixture && !suite.providerSuite?.fixtureModule) {
+        throw new Error(`${label}.input.workspaceFixture requires selected cases.suite.runtime.fixture.`);
+    }
+    const result = { ...row, id, split, tags };
+    Object.defineProperty(result, 'providerCase', { value: canonical, enumerable: false });
+    return result;
+}
+function parseCommonAudit(row, label, tags) {
+    const audit = row.audit === undefined ? undefined : record(row.audit, `${label}.audit`);
+    if (audit) {
+        onlyKeys(audit, ['rationale', 'errorCost', 'allowedVariance', 'minimalCounterexample'], `${label}.audit`);
+        for (const key of ['rationale', 'allowedVariance', 'minimalCounterexample']) {
+            if (Object.hasOwn(audit, key))
+                nonEmptyString(audit[key], `${label}.audit.${key}`);
+        }
+        if (audit.errorCost !== undefined
+            && audit.errorCost !== 'high'
+            && audit.errorCost !== 'medium'
+            && audit.errorCost !== 'low')
+            throw new Error(`${label}.audit.errorCost must be high, medium, or low.`);
+    }
+    const required = [
+        ...tags.some((tag) => tag === 'boundary' || tag === 'regression')
+            ? ['rationale', 'minimalCounterexample']
+            : [],
+        ...tags.some((tag) => tag === 'release-critical' || tag === 'high-cost') ? ['errorCost'] : [],
+        ...tags.includes('high-variance') ? ['allowedVariance'] : [],
+    ];
+    const missing = required.find((key) => !audit || !Object.hasOwn(audit, key));
+    if (missing)
+        throw new Error(`${label}.audit.${missing} is required by case tags.`);
+}
+function parseConversationCase(row, index) {
+    const label = `selected cases.cases[${index}]`;
+    onlyKeys(row, ['id', 'split', 'tags', 'prompt', 'assertions', 'input', 'expected', 'audit'], label);
+    const id = safeId(row.id, `${label}.id`);
+    const split = safeId(row.split, `${label}.split`);
+    const tags = uniqueStrings(row.tags, `${label}.tags`, true);
+    nonEmptyString(row.prompt, `${label}.prompt`);
+    if (!Array.isArray(row.assertions) || row.assertions.length === 0) {
+        throw new Error(`${label}.assertions must be a non-empty array.`);
+    }
+    const assertions = row.assertions.map((entry, assertionIndex) => nonEmptyString(entry, `${label}.assertions[${assertionIndex}]`));
+    const input = row.input === undefined ? {} : record(row.input, `${label}.input`);
+    onlyKeys(input, ['scriptedReplies', 'workspaceFiles', 'pathPrepend'], `${label}.input`);
+    if (input.scriptedReplies !== undefined) {
+        if (!Array.isArray(input.scriptedReplies)) {
+            throw new Error(`${label}.input.scriptedReplies must be an array.`);
+        }
+        input.scriptedReplies.forEach((reply, replyIndex) => {
+            const replyLabel = `${label}.input.scriptedReplies[${replyIndex}]`;
+            if (typeof reply === 'string')
+                return;
+            const replyRow = record(reply, replyLabel);
+            onlyKeys(replyRow, ['answersByIndex', 'expectedQuestionCount', 'unexpectedQuestionAnswer'], replyLabel);
+            if (!Array.isArray(replyRow.answersByIndex)
+                || replyRow.answersByIndex.some((answer) => typeof answer !== 'string'))
+                throw new Error(`${replyLabel}.answersByIndex must be an array of strings.`);
+            if (replyRow.expectedQuestionCount !== undefined
+                && (!Number.isInteger(replyRow.expectedQuestionCount)
+                    || replyRow.expectedQuestionCount < 0))
+                throw new Error(`${replyLabel}.expectedQuestionCount must be a non-negative integer.`);
+            nonEmptyString(replyRow.unexpectedQuestionAnswer, `${replyLabel}.unexpectedQuestionAnswer`);
+        });
+    }
+    if (input.workspaceFiles !== undefined) {
+        const workspaceFiles = record(input.workspaceFiles, `${label}.input.workspaceFiles`);
+        for (const [workspacePath, content] of Object.entries(workspaceFiles)) {
+            relativePath(workspacePath, `${label}.input.workspaceFiles.${workspacePath}`);
+            if (typeof content !== 'string') {
+                throw new Error(`${label}.input.workspaceFiles.${workspacePath} must be a string.`);
+            }
+        }
+    }
+    if (input.pathPrepend !== undefined) {
+        pathArray(input.pathPrepend, `${label}.input.pathPrepend`, { allowEmpty: true });
+    }
+    const expected = row.expected === undefined ? {} : record(row.expected, `${label}.expected`);
+    onlyKeys(expected, ['output', 'workspaceUnchanged', 'allowedWorkspaceChanges', 'artifacts'], `${label}.expected`);
+    if (expected.output !== undefined)
+        nonEmptyString(expected.output, `${label}.expected.output`);
+    if (expected.workspaceUnchanged !== undefined && typeof expected.workspaceUnchanged !== 'boolean') {
+        throw new Error(`${label}.expected.workspaceUnchanged must be boolean.`);
+    }
+    if (expected.allowedWorkspaceChanges !== undefined) {
+        pathArray(expected.allowedWorkspaceChanges, `${label}.expected.allowedWorkspaceChanges`, { allowEmpty: true, allowTreePattern: true });
+    }
+    if (expected.artifacts !== undefined) {
+        if (!Array.isArray(expected.artifacts))
+            throw new Error(`${label}.expected.artifacts must be an array.`);
+        optionalJson(expected.artifacts, `${label}.expected.artifacts`);
+        for (const [artifactIndex, artifact] of expected.artifacts.entries()) {
+            const artifactRow = record(artifact, `${label}.expected.artifacts[${artifactIndex}]`);
+            onlyKeys(artifactRow, ['path', 'exists', 'contains', 'notContains', 'json'], `${label}.expected.artifacts[${artifactIndex}]`);
+            relativePath(artifactRow.path, `${label}.expected.artifacts[${artifactIndex}].path`);
+        }
+    }
+    parseCommonAudit(row, label, tags);
+    return { ...row, id, split, tags, assertions };
+}
+function parseCase(value, index, suite) {
+    const label = `selected cases.cases[${index}]`;
+    const row = record(value, label);
+    const common = ['id', 'split', 'tags'];
+    if (suite.kind === 'trigger') {
+        const result = parseTriggerCase(row, index, suite);
+        parseCommonAudit(row, label, result.tags);
+        return result;
+    }
+    if (suite.kind === 'conversation')
+        return parseConversationCase(row, index);
+    onlyKeys(row, [...common, 'input', 'expected', 'assertions', 'audit'], label);
+    {
+        optionalJson(row.input, `${label}.input`);
+        optionalJson(row.expected, `${label}.expected`);
+        optionalJson(row.assertions, `${label}.assertions`);
+    }
+    const result = {
+        ...row,
+        id: safeId(row.id, `${label}.id`),
+        split: safeId(row.split, `${label}.split`),
+        tags: uniqueStrings(row.tags, `${label}.tags`, true),
+    };
+    parseCommonAudit(row, label, result.tags);
+    return result;
+}
+export function parseSelectedCases(value) {
+    const document = record(value, 'selected cases');
+    onlyKeys(document, ['version', 'suite', 'cases'], 'selected cases');
+    if (document.version !== 1)
+        throw new Error('selected cases.version must be 1.');
+    const suite = parseSuite(document.suite);
+    if (!Array.isArray(document.cases) || document.cases.length === 0) {
+        throw new Error('selected cases.cases must be a non-empty array.');
+    }
+    const cases = document.cases.map((entry, index) => parseCase(entry, index, suite));
+    if (new Set(cases.map((entry) => entry.id)).size !== cases.length) {
+        throw new Error('selected cases.cases must contain unique ids.');
+    }
+    return { version: 1, suite, cases };
+}
+export async function validateSelectedCases(value) {
+    try {
+        return { valid: true, errors: [], calculated: parseSelectedCases(value) };
+    }
+    catch (error) {
+        return { valid: false, errors: [error.message], calculated: null };
+    }
+}
+export async function validateSelectedCasesFile(casesPath) {
+    try {
+        const file = await readEvalContractFile(casesPath, 'selected cases');
+        return validateSelectedCases(file.value);
+    }
+    catch (error) {
+        return {
+            valid: false,
+            errors: [`Selected cases are unavailable or invalid: ${error.message}`],
+            calculated: null,
+        };
+    }
+}
+//# sourceMappingURL=selected-cases.js.map

package/dist/selected-cases.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"selected-cases.js","sourceRoot":"","sources":["../src/selected-cases.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAA;AACrD,OAAO,EACL,+BAA+B,GAGhC,MAAM,uBAAuB,CAAA;AAoC9B,SAAS,MAAM,CAAC,KAAc,EAAE,KAAa;IAC3C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAChE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,qBAAqB,CAAC,CAAA;IAChD,CAAC;IACD,OAAO,KAAgC,CAAA;AACzC,CAAC;AAED,SAAS,QAAQ,CAAC,KAA8B,EAAE,OAA0B,EAAE,KAAa;IACzF,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAA;IAC7B,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAChE,IAAI,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,sBAAsB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;AACxF,CAAC;AAED,SAAS,cAAc,CAAC,KAAc,EAAE,KAAa;IACnD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACrD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,8BAA8B,CAAC,CAAA;IACzD,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,MAAM,CAAC,KAAc,EAAE,KAAa;IAC3C,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;IAC3C,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,+BAA+B,CAAC,CAAA;IAC1D,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,KAAa,EAAE,UAAU,GAAG,KAAK;IACtE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,UAAU,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,YAAY,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,SAAS,CAAC,CAAA;IACjF,CAAC;IACD,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,cAAc,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,KAAK,GAAG,CAAC,CAAC,CAAA;IACvF,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,8BAA8B,CAAC,CAAA;IACnG,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,eAAe,CAAC,KAAc,EAAE,KAAa;IACpD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,OAAM;IACrF,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAM;IAC/D,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,KAAK,GAAG,CAAC,CAAC,CAAA;QAC7E,OAAM;IACR,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;IAChC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/C,IAAI,GAAG,KAAK,WAAW,IAAI,GAAG,KAAK,WAAW,IAAI,GAAG,KAAK,aAAa,EAAE,CAAC;YACxE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,0BAA0B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC3E,CAAC;QACD,eAAe,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,GAAG,EAAE,CAAC,CAAA;IAC3C,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,KAAc,EAAE,KAAa;IACjD,IAAI,KAAK,KAAK,SAAS;QAAE,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;IACtD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,YAAY,CACnB,KAAc,EACd,KAAa,EACb,UAA0C,EAAE;IAE5C,MAAM,GAAG,GAAG,cAAc,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;IACxC,MAAM,WAAW,GAAG,OAAO,CAAC,gBAAgB,KAAK,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IAC5E,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAA;IACtD,IACE,SAAS,KAAK,EAAE;WACb,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC;WACxB,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC;WACxB,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;WACvB,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;WACvB,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;WACvB,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;WACvB,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC;WACzB,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC;WAC7B,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,KAAK,EAAE,IAAI,OAAO,KAAK,GAAG,IAAI,OAAO,KAAK,IAAI,CAAC;QAChG,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,4CAA4C,CAAC,CAAA;IACvE,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,SAAS,SAAS,CAChB,KAAc,EACd,KAAa,EACb,UAAgE,EAAE;IAElE,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,UAAU,KAAK,IAAI,CAAC,CAAA;IACvE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,YAAY,CAC9C,KAAK,EACL,GAAG,KAAK,IAAI,KAAK,GAAG,EACpB,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAC/C,CAAC,CAAA;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAwB;IACpD,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,IAAI,EAAE,CAAA;IACnC,OAAO;QACL,EAAE,EAAE,UAAU;QACd,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,KAAK,CAAC,aAAa;QAC1B,MAAM,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,aAAa,EAAE,GAAG,CAAC,KAAK,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1E,KAAK,EAAE,YAAY;QACnB,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,KAAK,IAAI;QACrD,SAAS,EAAE,OAAO,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU;QACjE,oBAAoB,EAAE,CAAC,OAAO,CAAC,oBAAoB,IAAI,EAAE,CAA6C;QACtG,wBAAwB,EAAE,OAAO,CAAC,wBAAwB,KAAK,IAAI;QACnE,GAAG,CAAC,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClF,QAAQ,EAAE,CAAC,UAAU,CAAC;QACtB,WAAW,EAAE,CAAC;QACd,aAAa,EAAE,CAAC,KAAK,CAAC;QACtB,kBAAkB,EAAE,KAAK;QACzB,eAAe,EAAE,CAAC;KACnB,CAAA;AACH,CAAC;AAED,SAAS,YAAY,CACnB,KAAc,EACd,IAA2B;IAE3B,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,8BAA8B,CAAC,CAAA;IAC7D,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,QAAQ,CACN,OAAO,EACP;YACE,mBAAmB;YACnB,WAAW;YACX,sBAAsB;YACtB,0BAA0B;YAC1B,SAAS;YACT,iBAAiB;SAClB,EACD,8BAA8B,CAC/B,CAAA;QACD,IAAI,OAAO,CAAC,iBAAiB,KAAK,SAAS,IAAI,OAAO,OAAO,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;YAC9F,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAA;QACpF,CAAC;QACD,IAAI,OAAO,CAAC,wBAAwB,KAAK,SAAS,IAAI,OAAO,OAAO,CAAC,wBAAwB,KAAK,SAAS,EAAE,CAAC;YAC5G,MAAM,IAAI,KAAK,CAAC,wEAAwE,CAAC,CAAA;QAC3F,CAAC;QACD,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,OAAO,CAAC,SAAS,KAAK,UAAU,IAAI,OAAO,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;YAC1G,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAA;QACvF,CAAC;QACD,IAAI,OAAO,CAAC,oBAAoB,KAAK,SAAS,EAAE,CAAC;YAC/C,MAAM,MAAM,GAAG,aAAa,CAC1B,OAAO,CAAC,oBAAoB,EAC5B,mDAAmD,EACnD,IAAI,CACL,CAAA;YACD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,YAAY,IAAI,KAAK,KAAK,QAAQ,CAAC,CAAA;YACzG,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAA;YAC5F,CAAC;QACH,CAAC;QACD,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAClC,OAAO,CAAC,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,sCAAsC,CAAC,CAAA;QACzF,CAAC;IACH,CAAC;SAAM,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;QACnC,QAAQ,CAAC,OAAO,EAAE,CAAC,WAAW,EAAE,aAAa,EAAE,iBAAiB,CAAC,EAAE,8BAA8B,CAAC,CAAA;QAClG,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,OAAO,CAAC,SAAS,KAAK,UAAU,IAAI,OAAO,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;YAC1G,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAA;QACvF,CAAC;QACD,OAAO,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,KAAK,SAAS;YACrD,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,WAAW,EAAE,0CAA0C,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;IACtG,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,OAAO,EAAE,CAAC,iBAAiB,CAAC,EAAE,8BAA8B,CAAC,CAAA;IACxE,CAAC;IACD,IACE,OAAO,CAAC,eAAe,KAAK,SAAS;WAClC,CACD,OAAO,OAAO,CAAC,eAAe,KAAK,QAAQ;eACxC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC;eACzC,OAAO,CAAC,eAAe,GAAG,CAAC;eAC3B,OAAO,CAAC,eAAe,GAAG,CAAC,CAC/B,EACD,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,mFAAmF,CAAC,CAAA;IACtG,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,sBAAsB,CAAC,CAAA;IACnD,QAAQ,CACN,KAAK,EACL,CAAC,MAAM,EAAE,eAAe,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,CAAC,EAC9D,sBAAsB,CACvB,CAAA;IACD,IACE,KAAK,CAAC,IAAI,KAAK,SAAS;WACrB,KAAK,CAAC,IAAI,KAAK,cAAc;WAC7B,KAAK,CAAC,IAAI,KAAK,gBAAgB;WAC/B,KAAK,CAAC,IAAI,KAAK,gBAAgB,EAClC,CAAC;QACD,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAA;IACH,CAAC;IACD,MAAM,MAAM,GAAsB;QAChC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,aAAa,EAAE,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,oCAAoC,CAAC;KACjF,CAAA;IACD,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACrC,MAAM,CAAC,YAAY,GAAG,aAAa,CAAC,KAAK,CAAC,YAAY,EAAE,mCAAmC,CAAC;aACzF,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,qCAAqC,KAAK,GAAG,CAAC,CAAC,CAAA;IACxF,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,6BAA6B,CAAC,CAAA;QAClE,QAAQ,CACN,MAAM,EACN,CAAC,cAAc,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,CAAC,EACvE,6BAA6B,CAC9B,CAAA;QACD,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;YAC9E,GAAG;YACH,cAAc,CAAC,KAAK,EAAE,+BAA+B,GAAG,EAAE,CAAC;SAC5D,CAAC,CAAC,CAAA;IACL,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAChC,MAAM,CAAC,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAA;IAC1D,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC9B,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,eAAe,EAAE;YAC7C,KAAK,EAAE,oBAAoB,CAAC,MAAM,CAAC;YACnC,UAAU,EAAE,KAAK;SAClB,CAAC,CAAA;IACJ,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,kBAAkB,CACzB,GAA4B,EAC5B,KAAa;IAEb,IAAI,GAAG,CAAC,YAAY,KAAK,SAAS,IAAI,GAAG,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,mDAAmD,CAAC,CAAA;IAC9E,CAAC;IACD,IAAI,GAAG,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACnC,IAAI,OAAO,GAAG,CAAC,YAAY,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,gCAAgC,CAAC,CAAA;QACpG,OAAO,EAAE,iBAAiB,EAAE,GAAG,CAAC,YAAY,EAAE,CAAA;IAChD,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,KAAK,WAAW,CAAC,CAAA;IAC1D,QAAQ,CAAC,QAAQ,EAAE,CAAC,cAAc,CAAC,EAAE,GAAG,KAAK,WAAW,CAAC,CAAA;IACzD,MAAM,YAAY,GAAG,aAAa,CAAC,QAAQ,CAAC,YAAY,EAAE,GAAG,KAAK,wBAAwB,EAAE,IAAI,CAAC,CAAA;IACjG,OAAO,EAAE,sBAAsB,EAAE,YAAY,EAAE,CAAA;AACjD,CAAC;AAED,SAAS,gBAAgB,CACvB,GAA4B,EAC5B,KAAa,EACb,KAAwB;IAExB,MAAM,KAAK,GAAG,wBAAwB,KAAK,GAAG,CAAA;IAC9C,QAAQ,CACN,GAAG,EACH,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,EACzC,KAAK,CACN,CAAA;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,KAAK,QAAQ,CAAC,CAAA;IACjD,QAAQ,CACN,KAAK,EACL,CAAC,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,iBAAiB,CAAC,EAChG,GAAG,KAAK,QAAQ,CACjB,CAAA;IACD,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,KAAK,KAAK,CAAC,CAAA;IACxC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,KAAK,QAAQ,CAAC,CAAA;IACjD,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,KAAK,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3D,MAAM,SAAS,GAAG,+BAA+B,CAAC;QAChD,EAAE;QACF,KAAK;QACL,KAAK,EAAE,cAAc,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,KAAK,cAAc,CAAC;QAC1D,GAAG,kBAAkB,CAAC,KAAK,EAAE,GAAG,KAAK,QAAQ,CAAC;QAC9C,GAAG,CAAC,KAAK,CAAC,gBAAgB,KAAK,SAAS;YACtC,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,kBAAkB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC;QACnD,GAAG,CAAC,KAAK,CAAC,gBAAgB,KAAK,SAAS;YACtC,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,iBAAiB,EAAE,YAAY,CAAC,KAAK,CAAC,gBAAgB,EAAE,GAAG,KAAK,yBAAyB,CAAC,EAAE,CAAC;QACnG,GAAG,CAAC,KAAK,CAAC,eAAe,KAAK,SAAS;YACrC,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,iBAAiB,EAAE,KAAK,CAAC,eAAe,EAAE,CAAC;QACjD,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;KACvC,EAAE,KAAK,EAAE,KAAK,CAAC,aAAc,CAAC,CAAA;IAC/B,IAAI,SAAS,CAAC,iBAAiB,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,aAAa,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,wEAAwE,CAAC,CAAA;IACnG,CAAC;IACD,MAAM,MAAM,GAAG,EAAE,GAAG,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAsB,CAAA;IAC9D,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,cAAc,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAA;IACtF,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,gBAAgB,CAAC,GAA4B,EAAE,KAAa,EAAE,IAAc;IACnF,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,KAAK,QAAQ,CAAC,CAAA;IACvF,IAAI,KAAK,EAAE,CAAC;QACV,QAAQ,CACN,KAAK,EACL,CAAC,WAAW,EAAE,WAAW,EAAE,iBAAiB,EAAE,uBAAuB,CAAC,EACtE,GAAG,KAAK,QAAQ,CACjB,CAAA;QACD,KAAK,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,iBAAiB,EAAE,uBAAuB,CAAU,EAAE,CAAC;YACrF,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC;gBAAE,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,GAAG,KAAK,UAAU,GAAG,EAAE,CAAC,CAAA;QACpF,CAAC;QACD,IACE,KAAK,CAAC,SAAS,KAAK,SAAS;eAC1B,KAAK,CAAC,SAAS,KAAK,MAAM;eAC1B,KAAK,CAAC,SAAS,KAAK,QAAQ;eAC5B,KAAK,CAAC,SAAS,KAAK,KAAK;YAC5B,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,gDAAgD,CAAC,CAAA;IAC7E,CAAC;IACD,MAAM,QAAQ,GAAG;QACf,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,UAAU,IAAI,GAAG,KAAK,YAAY,CAAC;YAC/D,CAAC,CAAC,CAAC,WAAW,EAAE,uBAAuB,CAAC;YACxC,CAAC,CAAC,EAAE;QACN,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,kBAAkB,IAAI,GAAG,KAAK,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE;QAC7F,GAAG,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE;KAC7D,CAAA;IACD,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAA;IAC5E,IAAI,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,UAAU,OAAO,4BAA4B,CAAC,CAAA;AACrF,CAAC;AAED,SAAS,qBAAqB,CAC5B,GAA4B,EAC5B,KAAa;IAEb,MAAM,KAAK,GAAG,wBAAwB,KAAK,GAAG,CAAA;IAC9C,QAAQ,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,CAAA;IACnG,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,KAAK,KAAK,CAAC,CAAA;IACxC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,KAAK,QAAQ,CAAC,CAAA;IACjD,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,KAAK,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3D,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,KAAK,SAAS,CAAC,CAAA;IAC7C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,wCAAwC,CAAC,CAAA;IACnE,CAAC;IACD,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,CAAC,cAAc,CAC7E,KAAK,EACL,GAAG,KAAK,eAAe,cAAc,GAAG,CACzC,CAAC,CAAA;IACF,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,KAAK,QAAQ,CAAC,CAAA;IAChF,QAAQ,CAAC,KAAK,EAAE,CAAC,iBAAiB,EAAE,gBAAgB,EAAE,aAAa,CAAC,EAAE,GAAG,KAAK,QAAQ,CAAC,CAAA;IACvF,IAAI,KAAK,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QACxC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,0CAA0C,CAAC,CAAA;QACrE,CAAC;QACD,KAAK,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;YAClD,MAAM,UAAU,GAAG,GAAG,KAAK,0BAA0B,UAAU,GAAG,CAAA;YAClE,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,OAAM;YACrC,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,CAAA;YAC1C,QAAQ,CACN,QAAQ,EACR,CAAC,gBAAgB,EAAE,uBAAuB,EAAE,0BAA0B,CAAC,EACvE,UAAU,CACX,CAAA;YACD,IACE,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;mBACpC,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,MAAM,KAAK,QAAQ,CAAC;gBACvE,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,8CAA8C,CAAC,CAAA;YAC9E,IACE,QAAQ,CAAC,qBAAqB,KAAK,SAAS;mBACzC,CACD,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,qBAAqB,CAAC;uBAC7C,QAAQ,CAAC,qBAAgC,GAAG,CAAC,CAClD;gBACD,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,wDAAwD,CAAC,CAAA;YACxF,cAAc,CAAC,QAAQ,CAAC,wBAAwB,EAAE,GAAG,UAAU,2BAA2B,CAAC,CAAA;QAC7F,CAAC,CAAC,CAAA;IACJ,CAAC;IACD,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;QACvC,MAAM,cAAc,GAAG,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,KAAK,uBAAuB,CAAC,CAAA;QACpF,KAAK,MAAM,CAAC,aAAa,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;YACtE,YAAY,CAAC,aAAa,EAAE,GAAG,KAAK,yBAAyB,aAAa,EAAE,CAAC,CAAA;YAC7E,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;gBAChC,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,yBAAyB,aAAa,oBAAoB,CAAC,CAAA;YACrF,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACpC,SAAS,CAAC,KAAK,CAAC,WAAW,EAAE,GAAG,KAAK,oBAAoB,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;IAClF,CAAC;IACD,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,KAAK,WAAW,CAAC,CAAA;IAC5F,QAAQ,CACN,QAAQ,EACR,CAAC,QAAQ,EAAE,oBAAoB,EAAE,yBAAyB,EAAE,WAAW,CAAC,EACxE,GAAG,KAAK,WAAW,CACpB,CAAA;IACD,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS;QAAE,cAAc,CAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,KAAK,kBAAkB,CAAC,CAAA;IAC9F,IAAI,QAAQ,CAAC,kBAAkB,KAAK,SAAS,IAAI,OAAO,QAAQ,CAAC,kBAAkB,KAAK,SAAS,EAAE,CAAC;QAClG,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,+CAA+C,CAAC,CAAA;IAC1E,CAAC;IACD,IAAI,QAAQ,CAAC,uBAAuB,KAAK,SAAS,EAAE,CAAC;QACnD,SAAS,CACP,QAAQ,CAAC,uBAAuB,EAChC,GAAG,KAAK,mCAAmC,EAC3C,EAAE,UAAU,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAC7C,CAAA;IACH,CAAC;IACD,IAAI,QAAQ,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,uCAAuC,CAAC,CAAA;QACxG,YAAY,CAAC,QAAQ,CAAC,SAAS,EAAE,GAAG,KAAK,qBAAqB,CAAC,CAAA;QAC/D,KAAK,MAAM,CAAC,aAAa,EAAE,QAAQ,CAAC,IAAI,QAAQ,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;YACrE,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,EAAE,GAAG,KAAK,uBAAuB,aAAa,GAAG,CAAC,CAAA;YACrF,QAAQ,CACN,WAAW,EACX,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,CAAC,EACrD,GAAG,KAAK,uBAAuB,aAAa,GAAG,CAChD,CAAA;YACD,YAAY,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,KAAK,uBAAuB,aAAa,QAAQ,CAAC,CAAA;QACtF,CAAC;IACH,CAAC;IACD,gBAAgB,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,CAAA;IAClC,OAAO,EAAE,GAAG,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAsB,CAAA;AACpE,CAAC;AAED,SAAS,SAAS,CAAC,KAAc,EAAE,KAAa,EAAE,KAAwB;IACxE,MAAM,KAAK,GAAG,wBAAwB,KAAK,GAAG,CAAA;IAC9C,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;IAChC,MAAM,MAAM,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;IACtC,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,CAAA;QAClD,gBAAgB,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;QACzC,OAAO,MAAM,CAAA;IACf,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc;QAAE,OAAO,qBAAqB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;IAC3E,QAAQ,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,CAAA;IAC7E,CAAC;QACC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,KAAK,QAAQ,CAAC,CAAA;QACzC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,KAAK,WAAW,CAAC,CAAA;QAC/C,YAAY,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,KAAK,aAAa,CAAC,CAAA;IACrD,CAAC;IACD,MAAM,MAAM,GAAG;QACb,GAAG,GAAG;QACN,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,KAAK,KAAK,CAAC;QACjC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,KAAK,QAAQ,CAAC;QAC1C,IAAI,EAAE,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,KAAK,OAAO,EAAE,IAAI,CAAC;KACjC,CAAA;IACrB,gBAAgB,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;IACzC,OAAO,MAAM,CAAA;AACf,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAAc;IAC/C,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAA;IAChD,QAAQ,CAAC,QAAQ,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,gBAAgB,CAAC,CAAA;IACnE,IAAI,QAAQ,CAAC,OAAO,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAA;IAChF,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACxC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAA;IACpE,CAAC;IACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAA;IAClF,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAA;IAClE,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;AACrC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,KAAc;IACxD,IAAI,CAAC;QACH,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAA;IAC3E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,CAAE,KAAe,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAA;IAC/E,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,SAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,oBAAoB,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAA;QACpE,OAAO,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAC1C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,CAAC,8CAA+C,KAAe,CAAC,OAAO,EAAE,CAAC;YAClF,UAAU,EAAE,IAAI;SACjB,CAAA;IACH,CAAC;AACH,CAAC"}

package/dist/types.d.ts

@@ -4,25 +4,6 @@ export interface ValidationResult<TCalculated = unknown> {
     errors: string[];
     calculated: TCalculated | null;
 }
-export interface RuntimeCapabilities {
-    schemaVersion: 1;
-    provider: string;
-    providerVersion: string;
-    capabilities: {
-        triggerDiscovery: boolean;
-        skillPathExecution: boolean;
-        previousVersionBaseline: boolean;
-        withoutSkillBaseline: boolean;
-        cleanContext: boolean;
-        parallelRuns: boolean;
-        toolTrace: boolean;
-        tokenUsage: boolean;
-        duration: boolean;
-        sandbox: boolean;
-        materialization: boolean;
-    };
-    limitations: string[];
-}
 export interface RuntimeSkillSnapshot {
     root: string;
     digest: Digest;
@@ -32,6 +13,7 @@ export interface ProcessRequest {
     args: string[];
     cwd: string;
     env: NodeJS.ProcessEnv;
+    stdin?: string;
     timeoutMs: number;
     maxBufferBytes: number;
     signal?: AbortSignal;
@@ -69,18 +51,4 @@ export interface ProcessResult {
     containmentMode: 'managed' | 'verified-observed' | 'observed' | 'unmanaged-confirmed';
     containmentLimitation?: string;
 }
-export interface RunExecutionPlanOptions {
-    processHooks?: ProcessLifecycleHooks;
-    beforeBundleCommit?: () => void | Promise<void>;
-    sourceEnv?: NodeJS.ProcessEnv;
-    platform?: NodeJS.Platform;
-    processTree?: import('./process-tree.js').ProcessTreeDependencies;
-}
-export interface RunExecutionResult {
-    valid: true;
-    bundlePath: string;
-    bundleDigest: Digest;
-    evidenceLevel: 'executed' | 'comparative';
-    requirementsMet: boolean;
-}
-export type SchemaName = 'aw-skill-eval-execution-plan' | 'eval-run-bundle' | 'eval-runtime-capabilities';
+export type SchemaName = 'eval-spec' | 'eval-audit' | 'eval-result';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lesliechan721/agent-skill-eval",
-  "version": "0.1.0-beta.7",
+  "version": "0.1.0-beta.9",
   "type": "module",
   "engines": {
     "node": ">=22"
@@ -21,7 +21,12 @@
   "files": [
     "dist/"
   ],
+  "dependencies": {
+    "fs-extra": "^11.3.0",
+    "toml-eslint-parser": "^1.0.3"
+  },
   "devDependencies": {
+    "@types/fs-extra": "^11.0.4",
     "@types/node": "^24.0.0",
     "typescript": "^5.9.3",
     "vitest": "^3.2.4"