@lesliechan721/agent-skill-eval 0.1.0-beta.1

package/dist/runtime-skill.js

@@ -0,0 +1,135 @@
+import { createHash } from 'node:crypto';
+import { chmod, copyFile, lstat, mkdir, readFile, readdir, realpath, stat, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { pathsOverlap } from './common.js';
+const alwaysIgnored = new Set(['.git', 'node_modules', '.DS_Store']);
+const authorOnlyRoots = new Set(['evals', 'reports']);
+const productionDigestDomain = 'aw-production-records-v1';
+function normalizedRegularFileMode(mode, platform) {
+    // Windows does not expose a portable executable bit, so its stable runtime mode is 100644.
+    return platform !== 'win32' && (mode & 0o111) !== 0 ? '100755' : '100644';
+}
+function comparePosixPaths(left, right) {
+    return left < right ? -1 : left > right ? 1 : 0;
+}
+function updateLengthPrefixed(hash, value) {
+    const bytes = typeof value === 'string' ? Buffer.from(value, 'utf8') : value;
+    const length = Buffer.allocUnsafe(8);
+    length.writeBigUInt64BE(BigInt(bytes.length));
+    hash.update(length);
+    hash.update(bytes);
+}
+function digestProductionRecords(files) {
+    const hash = createHash('sha256');
+    updateLengthPrefixed(hash, productionDigestDomain);
+    for (const file of [...files].sort((left, right) => comparePosixPaths(left.relativePath, right.relativePath))) {
+        updateLengthPrefixed(hash, file.relativePath);
+        updateLengthPrefixed(hash, file.mode);
+        updateLengthPrefixed(hash, createHash('sha256').update(file.bytes).digest());
+    }
+    return `sha256:${hash.digest('hex')}`;
+}
+async function runtimeSkillFiles(root, platform) {
+    const rootStat = await stat(root);
+    if (rootStat.isFile()) {
+        return [{
+                absolutePath: root,
+                relativePath: path.basename(root),
+                mode: normalizedRegularFileMode(rootStat.mode, platform),
+            }];
+    }
+    if (!rootStat.isDirectory())
+        throw new Error(`Skill target is not a file or directory: ${root}`);
+    const files = [];
+    async function walk(directory, relativeDirectory = '') {
+        const entries = await readdir(directory, { withFileTypes: true });
+        entries.sort((left, right) => left.name.localeCompare(right.name));
+        for (const entry of entries) {
+            if (alwaysIgnored.has(entry.name))
+                continue;
+            if (relativeDirectory === '' && authorOnlyRoots.has(entry.name))
+                continue;
+            const absolutePath = path.join(directory, entry.name);
+            const relativePath = path.join(relativeDirectory, entry.name);
+            if (entry.isSymbolicLink())
+                throw new Error(`Skill target cannot contain symlinks: ${relativePath}`);
+            if (entry.isDirectory())
+                await walk(absolutePath, relativePath);
+            else if (entry.isFile()) {
+                const fileStat = await stat(absolutePath);
+                files.push({
+                    absolutePath,
+                    relativePath: relativePath.split(path.sep).join('/'),
+                    mode: normalizedRegularFileMode(fileStat.mode, platform),
+                });
+            }
+        }
+    }
+    await walk(root);
+    return files;
+}
+export async function computeSkillDigest(skillPath, options = {}) {
+    if ((await lstat(skillPath)).isSymbolicLink())
+        throw new Error('Skill target cannot be a symlink');
+    const canonicalRoot = await realpath(skillPath);
+    const files = await Promise.all((await runtimeSkillFiles(canonicalRoot, options.platform ?? process.platform)).map(async (file) => ({
+        relativePath: file.relativePath,
+        mode: file.mode,
+        bytes: await readFile(file.absolutePath),
+    })));
+    return digestProductionRecords(files);
+}
+function digestCapturedRuntimeSkill(files) {
+    return digestProductionRecords(files);
+}
+export async function captureRuntimeSkill(skillPath, options = {}) {
+    if ((await lstat(skillPath)).isSymbolicLink())
+        throw new Error('Skill target cannot be a symlink');
+    const sourceRoot = await realpath(skillPath);
+    const files = await Promise.all((await runtimeSkillFiles(sourceRoot, options.platform ?? process.platform)).map(async (file) => ({
+        relativePath: file.relativePath,
+        mode: file.mode,
+        bytes: await readFile(file.absolutePath),
+    })));
+    return { sourceRoot, digest: digestCapturedRuntimeSkill(files), files };
+}
+export async function materializeCapturedRuntimeSkill(captured, destinationPath, options = {}) {
+    const resolvedDestination = path.resolve(destinationPath);
+    if (pathsOverlap(captured.sourceRoot, resolvedDestination))
+        throw new Error('Runtime Skill snapshot destination must not overlap the source Skill');
+    await mkdir(destinationPath, { recursive: true });
+    const platform = options.platform ?? process.platform;
+    for (const file of captured.files) {
+        const destination = path.join(destinationPath, ...file.relativePath.split('/'));
+        await mkdir(path.dirname(destination), { recursive: true });
+        await writeFile(destination, file.bytes, { flag: 'wx', mode: file.mode === '100755' ? 0o755 : 0o644 });
+        if (platform !== 'win32')
+            await chmod(destination, file.mode === '100755' ? 0o755 : 0o644);
+    }
+    const digest = await computeSkillDigest(destinationPath, options);
+    if (digest !== captured.digest)
+        throw new Error('Runtime Skill snapshot digest does not match captured Skill bytes');
+    return { root: await realpath(destinationPath), digest };
+}
+export async function createRuntimeSkillSnapshot(sourcePath, destinationPath, options = {}) {
+    if ((await lstat(sourcePath)).isSymbolicLink())
+        throw new Error('Skill target cannot be a symlink');
+    const canonicalSource = await realpath(sourcePath);
+    const resolvedDestination = path.resolve(destinationPath);
+    if (pathsOverlap(canonicalSource, resolvedDestination))
+        throw new Error('Runtime Skill snapshot destination must not overlap the source Skill');
+    await mkdir(destinationPath, { recursive: true });
+    const platform = options.platform ?? process.platform;
+    for (const file of await runtimeSkillFiles(canonicalSource, platform)) {
+        const destination = path.join(destinationPath, ...file.relativePath.split('/'));
+        await mkdir(path.dirname(destination), { recursive: true });
+        await copyFile(file.absolutePath, destination);
+        if (platform !== 'win32')
+            await chmod(destination, file.mode === '100755' ? 0o755 : 0o644);
+    }
+    return {
+        root: await realpath(destinationPath),
+        digest: await computeSkillDigest(destinationPath, options),
+    };
+}
+//# sourceMappingURL=runtime-skill.js.map

package/dist/runtime-skill.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runtime-skill.js","sourceRoot":"","sources":["../src/runtime-skill.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC9G,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAG1C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,EAAE,WAAW,CAAC,CAAC,CAAA;AACpE,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAA;AACrD,MAAM,sBAAsB,GAAG,0BAA0B,CAAA;AAwBzD,SAAS,yBAAyB,CAAC,IAAY,EAAE,QAAyB;IACxE,2FAA2F;IAC3F,OAAO,QAAQ,KAAK,OAAO,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAA;AAC3E,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,KAAa;IACpD,OAAO,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;AACjD,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAmC,EAAE,KAAsB;IACvF,MAAM,KAAK,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;IAC5E,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAA;IACpC,MAAM,CAAC,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;IAC7C,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IACnB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AACpB,CAAC;AAED,SAAS,uBAAuB,CAC9B,KAAwF;IAExF,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAA;IACjC,oBAAoB,CAAC,IAAI,EAAE,sBAAsB,CAAC,CAAA;IAClD,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACjD,iBAAiB,CAAC,IAAI,CAAC,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC;QAC5D,oBAAoB,CAAC,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,CAAA;QAC7C,oBAAoB,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;QACrC,oBAAoB,CAAC,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;IAC9E,CAAC;IACD,OAAO,UAAU,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;AACvC,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,IAAY,EACZ,QAAyB;IAEzB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;QACtB,OAAO,CAAC;gBACN,YAAY,EAAE,IAAI;gBAClB,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACjC,IAAI,EAAE,yBAAyB,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;aACzD,CAAC,CAAA;IACJ,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,IAAI,EAAE,CAAC,CAAA;IAEhG,MAAM,KAAK,GAAuB,EAAE,CAAA;IACpC,KAAK,UAAU,IAAI,CAAC,SAAiB,EAAE,iBAAiB,GAAG,EAAE;QAC3D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAA;QACjE,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;QAClE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,SAAQ;YAC3C,IAAI,iBAAiB,KAAK,EAAE,IAAI,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,SAAQ;YACzE,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAA;YACrD,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,CAAC,IAAI,CAAC,CAAA;YAC7D,IAAI,KAAK,CAAC,cAAc,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,YAAY,EAAE,CAAC,CAAA;YACpG,IAAI,KAAK,CAAC,WAAW,EAAE;gBAAE,MAAM,IAAI,CAAC,YAAY,EAAE,YAAY,CAAC,CAAA;iBAC1D,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;gBACxB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,CAAA;gBACzC,KAAK,CAAC,IAAI,CAAC;oBACT,YAAY;oBACZ,YAAY,EAAE,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;oBACpD,IAAI,EAAE,yBAAyB,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;iBACzD,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,IAAI,CAAC,IAAI,CAAC,CAAA;IAChB,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,SAAiB,EACjB,UAAqC,EAAE;IAEvC,IAAI,CAAC,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;IAClG,MAAM,aAAa,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAA;IAC/C,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,CAAC,MAAM,iBAAiB,CAAC,aAAa,EAAE,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAChF,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;QACf,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,KAAK,EAAE,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC;KACzC,CAAC,CACH,CACF,CAAA;IACD,OAAO,uBAAuB,CAAC,KAAK,CAAC,CAAA;AACvC,CAAC;AAED,SAAS,0BAA0B,CAAC,KAAiC;IACnE,OAAO,uBAAuB,CAAC,KAAK,CAAC,CAAA;AACvC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,SAAiB,EACjB,UAAqC,EAAE;IAEvC,IAAI,CAAC,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;IAClG,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAA;IAC5C,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,CAAC,MAAM,iBAAiB,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;QAC/F,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,KAAK,EAAE,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC;KACzC,CAAC,CAAC,CACJ,CAAA;IACD,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,0BAA0B,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAA;AACzE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,+BAA+B,CACnD,QAA8B,EAC9B,eAAuB,EACvB,UAAqC,EAAE;IAEvC,MAAM,mBAAmB,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;IACzD,IAAI,YAAY,CAAC,QAAQ,CAAC,UAAU,EAAE,mBAAmB,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAA;IACzF,MAAM,KAAK,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACjD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAA;IACrD,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAA;QAC/E,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAC3D,MAAM,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAA;QACtG,IAAI,QAAQ,KAAK,OAAO;YAAE,MAAM,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IAC5F,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,eAAe,EAAE,OAAO,CAAC,CAAA;IACjE,IAAI,MAAM,KAAK,QAAQ,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAA;IACpH,OAAO,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAA;AAC1D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,UAAkB,EAClB,eAAuB,EACvB,UAAqC,EAAE;IAEvC,IAAI,CAAC,MAAM,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;IACnG,MAAM,eAAe,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,CAAA;IAClD,MAAM,mBAAmB,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;IACzD,IAAI,YAAY,CAAC,eAAe,EAAE,mBAAmB,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAA;IACzF,MAAM,KAAK,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACjD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAA;IACrD,KAAK,MAAM,IAAI,IAAI,MAAM,iBAAiB,CAAC,eAAe,EAAE,QAAQ,CAAC,EAAE,CAAC;QACtE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAA;QAC/E,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAC3D,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,WAAW,CAAC,CAAA;QAC9C,IAAI,QAAQ,KAAK,OAAO;YAAE,MAAM,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IAC5F,CAAC;IACD,OAAO;QACL,IAAI,EAAE,MAAM,QAAQ,CAAC,eAAe,CAAC;QACrC,MAAM,EAAE,MAAM,kBAAkB,CAAC,eAAe,EAAE,OAAO,CAAC;KAC3D,CAAA;AACH,CAAC"}

package/dist/schemas/aw-skill-eval-execution-plan.schema.json

@@ -0,0 +1,220 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agent-skill-toolkit.invalid/schemas/aw-skill-eval-execution-plan.schema.json",
+  "title": "AwSkillEvalExecutionPlan",
+  "type": "object",
+  "required": [
+    "schemaVersion",
+    "planId",
+    "target",
+    "cases",
+    "provider",
+    "configurations",
+    "runsPerCase",
+    "timeoutMs",
+    "outputRoot",
+    "capture"
+  ],
+  "properties": {
+    "schemaVersion": {
+      "const": 3
+    },
+    "planId": {
+      "type": "string",
+      "minLength": 1,
+      "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$"
+    },
+    "minimumEvidenceLevel": {
+      "enum": [
+        "executed",
+        "comparative"
+      ]
+    },
+    "target": {
+      "type": "object",
+      "required": [
+        "name",
+        "path",
+        "digest"
+      ],
+      "properties": {
+        "name": {
+          "type": "string",
+          "minLength": 1
+        },
+        "path": {
+          "type": "string",
+          "minLength": 1
+        },
+        "digest": {
+          "type": "string",
+          "pattern": "^sha256:[0-9a-f]{64}$"
+        }
+      },
+      "additionalProperties": false
+    },
+    "cases": {
+      "type": "object",
+      "required": [
+        "paths",
+        "corpusDigest"
+      ],
+      "properties": {
+        "paths": {
+          "type": "array",
+          "minItems": 1,
+          "items": {
+            "type": "string",
+            "minLength": 1
+          },
+          "uniqueItems": true
+        },
+        "corpusDigest": {
+          "type": "string",
+          "pattern": "^sha256:[0-9a-f]{64}$"
+        }
+      },
+      "additionalProperties": false
+    },
+    "provider": {
+      "type": "object",
+      "required": [
+        "id",
+        "command",
+        "capabilitiesPath",
+        "capabilitiesDigest",
+        "inheritEnvironment"
+      ],
+      "properties": {
+        "id": {
+          "type": "string",
+          "minLength": 1
+        },
+        "command": {
+          "type": "array",
+          "minItems": 1,
+          "items": {
+            "type": "string",
+            "minLength": 1
+          }
+        },
+        "cwd": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "env": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "string"
+          }
+        },
+        "capabilitiesPath": {
+          "type": "string",
+          "minLength": 1
+        },
+        "capabilitiesDigest": {
+          "type": "string",
+          "pattern": "^sha256:[0-9a-f]{64}$"
+        },
+        "inheritEnvironment": {
+          "type": "boolean"
+        }
+      },
+      "additionalProperties": false
+    },
+    "configurations": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "required": [
+          "id",
+          "skillMode",
+          "comparisonId"
+        ],
+        "properties": {
+          "id": {
+            "type": "string",
+            "minLength": 1,
+            "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$"
+          },
+          "skillMode": {
+            "enum": [
+              "with-skill",
+              "without-skill",
+              "previous-version"
+            ]
+          },
+          "comparisonId": {
+            "type": [
+              "string",
+              "null"
+            ],
+            "minLength": 1,
+            "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$"
+          },
+          "skillPath": {
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "skillDigest": {
+            "type": [
+              "string",
+              "null"
+            ],
+            "pattern": "^sha256:[0-9a-f]{64}$"
+          },
+          "env": {
+            "type": "object",
+            "additionalProperties": {
+              "type": "string"
+            }
+          }
+        },
+        "additionalProperties": false
+      }
+    },
+    "runsPerCase": {
+      "type": "integer",
+      "minimum": 1
+    },
+    "timeoutMs": {
+      "type": "integer",
+      "minimum": 1000
+    },
+    "outputRoot": {
+      "type": "string",
+      "minLength": 1,
+      "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$))(?!.*\\\\).+$"
+    },
+    "capture": {
+      "type": "object",
+      "required": [
+        "outputs",
+        "trace",
+        "timing",
+        "usage"
+      ],
+      "properties": {
+        "outputs": {
+          "type": "boolean"
+        },
+        "trace": {
+          "type": "boolean"
+        },
+        "timing": {
+          "type": "boolean"
+        },
+        "usage": {
+          "type": "boolean"
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+  "additionalProperties": false
+}

package/dist/schemas/digests.json

@@ -0,0 +1,5 @@
+{
+  "aw-skill-eval-execution-plan": "sha256:39ffdc4d731c5d98927f130e3c728e31b34dbc538eb8676a17ca3be8d54bd1ef",
+  "eval-run-bundle": "sha256:eb245510b130141ec129f13c5e18b44679e2a3d77e071c30d0b614ef052b35ed",
+  "eval-runtime-capabilities": "sha256:a63f1a1f4be0339253566050496b40cc9d42f4d1664f20205cb2614343455bd5"
+}