@lesliechan721/agent-skill-eval 0.1.0-beta.1

package/dist/schemas/eval-run-bundle.schema.json

@@ -0,0 +1,408 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agent-skill-toolkit.invalid/schemas/eval-run-bundle.schema.json",
+  "title": "EvalRunBundle",
+  "type": "object",
+  "required": [
+    "schemaVersion",
+    "runner",
+    "independentContext",
+    "provider",
+    "planDigest",
+    "targetSkillDigest",
+    "caseCorpusDigest",
+    "comparisonPlanned",
+    "evidenceLevel",
+    "runs",
+    "aggregate",
+    "limitations"
+  ],
+  "properties": {
+    "schemaVersion": {
+      "const": 3
+    },
+    "runner": {
+      "type": "object",
+      "required": [
+        "id",
+        "version"
+      ],
+      "properties": {
+        "id": {
+          "type": "string",
+          "minLength": 1
+        },
+        "version": {
+          "type": "string",
+          "minLength": 1
+        }
+      },
+      "additionalProperties": false
+    },
+    "independentContext": {
+      "type": "boolean"
+    },
+    "provider": {
+      "type": "object",
+      "required": [
+        "id",
+        "version"
+      ],
+      "properties": {
+        "id": {
+          "type": "string",
+          "minLength": 1
+        },
+        "version": {
+          "type": "string",
+          "minLength": 1
+        }
+      },
+      "additionalProperties": false
+    },
+    "planDigest": {
+      "type": "string",
+      "pattern": "^sha256:[0-9a-f]{64}$"
+    },
+    "targetSkillDigest": {
+      "type": "string",
+      "pattern": "^sha256:[0-9a-f]{64}$"
+    },
+    "caseCorpusDigest": {
+      "type": "string",
+      "pattern": "^sha256:[0-9a-f]{64}$"
+    },
+    "comparisonPlanned": {
+      "type": "boolean"
+    },
+    "evidenceLevel": {
+      "enum": [
+        "executed",
+        "comparative"
+      ]
+    },
+    "evidenceRequirement": {
+      "type": "object",
+      "required": [
+        "minimumEvidenceLevel",
+        "plannedComparisons",
+        "comparableComparisons",
+        "satisfied"
+      ],
+      "properties": {
+        "minimumEvidenceLevel": {
+          "enum": [
+            "executed",
+            "comparative"
+          ]
+        },
+        "plannedComparisons": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "comparableComparisons": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "satisfied": {
+          "type": "boolean"
+        }
+      },
+      "additionalProperties": false
+    },
+    "processManagement": {
+      "type": "object",
+      "required": [
+        "mode",
+        "descendantCleanupGuaranteed"
+      ],
+      "properties": {
+        "mode": {
+          "enum": [
+            "managed",
+            "verified-observed",
+            "observed"
+          ]
+        },
+        "descendantCleanupGuaranteed": {
+          "type": "boolean"
+        },
+        "limitation": {
+          "type": "string",
+          "minLength": 1
+        }
+      },
+      "allOf": [
+        {
+          "if": {
+            "properties": {
+              "mode": {
+                "enum": [
+                  "verified-observed",
+                  "observed"
+                ]
+              }
+            },
+            "required": [
+              "mode"
+            ]
+          },
+          "then": {
+            "required": [
+              "limitation"
+            ]
+          }
+        }
+      ],
+      "additionalProperties": false
+    },
+    "runs": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "required": [
+          "caseId",
+          "configuration",
+          "comparisonId",
+          "skillMode",
+          "skillDigest",
+          "runIndex",
+          "status",
+          "exitCode",
+          "durationMs",
+          "stdoutPath",
+          "stdoutDigest",
+          "stderrPath",
+          "stderrDigest",
+          "usage",
+          "grading",
+          "artifacts"
+        ],
+        "properties": {
+          "caseId": {
+            "type": "string",
+            "minLength": 1
+          },
+          "configuration": {
+            "type": "string",
+            "minLength": 1
+          },
+          "comparisonId": {
+            "type": [
+              "string",
+              "null"
+            ],
+            "minLength": 1
+          },
+          "skillMode": {
+            "enum": [
+              "with-skill",
+              "without-skill",
+              "previous-version"
+            ]
+          },
+          "skillDigest": {
+            "type": [
+              "string",
+              "null"
+            ],
+            "pattern": "^sha256:[0-9a-f]{64}$"
+          },
+          "runIndex": {
+            "type": "integer",
+            "minimum": 1
+          },
+          "status": {
+            "enum": [
+              "completed",
+              "process-failed",
+              "timeout",
+              "error"
+            ]
+          },
+          "exitCode": {
+            "type": [
+              "integer",
+              "null"
+            ]
+          },
+          "durationMs": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "stdoutPath": {
+            "type": "string",
+            "minLength": 1
+          },
+          "stdoutDigest": {
+            "type": "string",
+            "pattern": "^sha256:[0-9a-f]{64}$"
+          },
+          "stderrPath": {
+            "type": "string",
+            "minLength": 1
+          },
+          "stderrDigest": {
+            "type": "string",
+            "pattern": "^sha256:[0-9a-f]{64}$"
+          },
+          "usage": {
+            "type": [
+              "object",
+              "null"
+            ]
+          },
+          "grading": {
+            "type": [
+              "object",
+              "null"
+            ],
+            "required": [
+              "grader",
+              "result"
+            ],
+            "properties": {
+              "grader": {
+                "type": "object",
+                "required": [
+                  "id",
+                  "version"
+                ],
+                "properties": {
+                  "id": {
+                    "type": "string",
+                    "minLength": 1
+                  },
+                  "version": {
+                    "type": "string",
+                    "minLength": 1
+                  }
+                },
+                "additionalProperties": true
+              },
+              "result": {
+                "type": "object"
+              }
+            },
+            "additionalProperties": true
+          },
+          "artifacts": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "required": [
+                "role",
+                "path",
+                "digest"
+              ],
+              "properties": {
+                "role": {
+                  "enum": [
+                    "trace",
+                    "report",
+                    "output",
+                    "grading",
+                    "other"
+                  ]
+                },
+                "path": {
+                  "type": "string",
+                  "minLength": 1
+                },
+                "digest": {
+                  "type": "string",
+                  "pattern": "^sha256:[0-9a-f]{64}$"
+                }
+              },
+              "additionalProperties": false
+            }
+          }
+        },
+        "additionalProperties": false
+      }
+    },
+    "aggregate": {
+      "type": "object",
+      "required": [
+        "cases",
+        "configurations",
+        "runs",
+        "completed",
+        "processFailed",
+        "timedOut",
+        "comparisons"
+      ],
+      "properties": {
+        "cases": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "configurations": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "runs": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "completed": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "processFailed": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "timedOut": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "comparisons": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": [
+              "comparisonId",
+              "caseId",
+              "runIndex",
+              "paired",
+              "comparable"
+            ],
+            "properties": {
+              "comparisonId": {
+                "type": "string",
+                "minLength": 1
+              },
+              "caseId": {
+                "type": "string",
+                "minLength": 1
+              },
+              "runIndex": {
+                "type": "integer",
+                "minimum": 1
+              },
+              "paired": {
+                "type": "boolean"
+              },
+              "comparable": {
+                "type": "boolean"
+              }
+            },
+            "additionalProperties": false
+          }
+        }
+      },
+      "additionalProperties": false
+    },
+    "limitations": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "minLength": 1
+      },
+      "uniqueItems": true
+    }
+  },
+  "additionalProperties": false
+}

package/dist/schemas/eval-runtime-capabilities.schema.json

@@ -0,0 +1,85 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agent-skill-toolkit.invalid/schemas/eval-runtime-capabilities.schema.json",
+  "title": "EvalRuntimeCapabilities",
+  "type": "object",
+  "required": [
+    "schemaVersion",
+    "provider",
+    "providerVersion",
+    "capabilities",
+    "limitations"
+  ],
+  "properties": {
+    "schemaVersion": {
+      "const": 1
+    },
+    "provider": {
+      "type": "string",
+      "minLength": 1
+    },
+    "providerVersion": {
+      "type": "string",
+      "minLength": 1
+    },
+    "capabilities": {
+      "type": "object",
+      "required": [
+        "triggerDiscovery",
+        "skillPathExecution",
+        "previousVersionBaseline",
+        "withoutSkillBaseline",
+        "cleanContext",
+        "parallelRuns",
+        "toolTrace",
+        "tokenUsage",
+        "duration",
+        "sandbox",
+        "materialization"
+      ],
+      "properties": {
+        "triggerDiscovery": {
+          "type": "boolean"
+        },
+        "skillPathExecution": {
+          "type": "boolean"
+        },
+        "previousVersionBaseline": {
+          "type": "boolean"
+        },
+        "withoutSkillBaseline": {
+          "type": "boolean"
+        },
+        "cleanContext": {
+          "type": "boolean"
+        },
+        "parallelRuns": {
+          "type": "boolean"
+        },
+        "toolTrace": {
+          "type": "boolean"
+        },
+        "tokenUsage": {
+          "type": "boolean"
+        },
+        "duration": {
+          "type": "boolean"
+        },
+        "sandbox": {
+          "type": "boolean"
+        },
+        "materialization": {
+          "type": "boolean"
+        }
+      },
+      "additionalProperties": false
+    },
+    "limitations": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    }
+  },
+  "additionalProperties": false
+}

package/dist/schemas.d.ts

@@ -0,0 +1,5 @@
+import type { Digest, RuntimeCapabilities, SchemaName, ValidationResult } from './types.js';
+export declare function isSchemaName(value: string): value is SchemaName;
+export declare function getSchema(name: SchemaName): Promise<Record<string, unknown>>;
+export declare function getSchemaDigest(name: SchemaName): Promise<Digest>;
+export declare function validateRuntimeCapabilities(value: unknown): Promise<ValidationResult<RuntimeCapabilities>>;

package/dist/schemas.js

@@ -0,0 +1,27 @@
+import { readFile } from 'node:fs/promises';
+import { validateJsonSchema } from './json-schema.js';
+const schemaNames = new Set([
+    'aw-skill-eval-execution-plan',
+    'eval-run-bundle',
+    'eval-runtime-capabilities',
+]);
+export function isSchemaName(value) {
+    return schemaNames.has(value);
+}
+export async function getSchema(name) {
+    const source = new URL(`./schemas/${name}.schema.json`, import.meta.url);
+    return JSON.parse(await readFile(source, 'utf8'));
+}
+export async function getSchemaDigest(name) {
+    const digests = JSON.parse(await readFile(new URL('./schemas/digests.json', import.meta.url), 'utf8'));
+    return digests[name];
+}
+export async function validateRuntimeCapabilities(value) {
+    const errors = validateJsonSchema(await getSchema('eval-runtime-capabilities'), value);
+    return {
+        valid: errors.length === 0,
+        errors,
+        calculated: errors.length === 0 ? value : null,
+    };
+}
+//# sourceMappingURL=schemas.js.map

package/dist/schemas.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schemas.js","sourceRoot":"","sources":["../src/schemas.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAQrD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAa;IACtC,8BAA8B;IAC9B,iBAAiB;IACjB,2BAA2B;CAC5B,CAAC,CAAA;AAEF,MAAM,UAAU,YAAY,CAAC,KAAa;IACxC,OAAO,WAAW,CAAC,GAAG,CAAC,KAAmB,CAAC,CAAA;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAgB;IAC9C,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,aAAa,IAAI,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACxE,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAA4B,CAAA;AAC9E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAgB;IACpD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CACxB,MAAM,QAAQ,CAAC,IAAI,GAAG,CAAC,wBAAwB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAC7C,CAAA;IAC/B,OAAO,OAAO,CAAC,IAAI,CAAC,CAAA;AACtB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,KAAc;IAEd,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,SAAS,CAAC,2BAA2B,CAAC,EAAE,KAAK,CAAC,CAAA;IACtF,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;QAC1B,MAAM;QACN,UAAU,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAA4B,CAAC,CAAC,CAAC,IAAI;KACtE,CAAA;AACH,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,86 @@
+export type Digest = `sha256:${string}`;
+export interface ValidationResult<TCalculated = unknown> {
+    valid: boolean;
+    errors: string[];
+    calculated: TCalculated | null;
+}
+export interface RuntimeCapabilities {
+    schemaVersion: 1;
+    provider: string;
+    providerVersion: string;
+    capabilities: {
+        triggerDiscovery: boolean;
+        skillPathExecution: boolean;
+        previousVersionBaseline: boolean;
+        withoutSkillBaseline: boolean;
+        cleanContext: boolean;
+        parallelRuns: boolean;
+        toolTrace: boolean;
+        tokenUsage: boolean;
+        duration: boolean;
+        sandbox: boolean;
+        materialization: boolean;
+    };
+    limitations: string[];
+}
+export interface RuntimeSkillSnapshot {
+    root: string;
+    digest: Digest;
+}
+export interface ProcessRequest {
+    executable: string;
+    args: string[];
+    cwd: string;
+    env: NodeJS.ProcessEnv;
+    timeoutMs: number;
+    maxBufferBytes: number;
+    signal?: AbortSignal;
+}
+export interface ProviderProcessHandle {
+    pid: number;
+}
+export type ProcessTerminationReason = 'complete' | 'timeout' | 'abort' | 'max-buffer';
+export interface ProcessTreeCleanupConfirmation {
+    confirmed: true;
+}
+export interface ExternalContainmentConfirmation {
+    confirmed: true;
+    descendantCleanupGuaranteed: true;
+    readOnlySkillSnapshotGuaranteed: true;
+}
+export interface ProcessLifecycleHooks {
+    externalContainment?: ExternalContainmentConfirmation;
+    onSpawn?(handle: ProviderProcessHandle): void | Promise<void>;
+    terminateProcessTree?(handle: ProviderProcessHandle, reason: ProcessTerminationReason): Promise<ProcessTreeCleanupConfirmation>;
+}
+export interface ProcessResult {
+    stdout: string;
+    stderr: string;
+    exitCode: number | null;
+    signal: NodeJS.Signals | null;
+    timedOut: boolean;
+    cancelled: boolean;
+    terminationReason: ProcessTerminationReason | null;
+    maxBufferExceeded: boolean;
+    stdoutTruncated: boolean;
+    stderrTruncated: boolean;
+    descendantCleanupGuaranteed: boolean;
+    gradingEvidenceAccepted: boolean;
+    containmentMode: 'managed' | 'verified-observed' | 'observed' | 'unmanaged-confirmed';
+    containmentLimitation?: string;
+}
+export interface RunExecutionPlanOptions {
+    processHooks?: ProcessLifecycleHooks;
+    beforeBundleCommit?: () => void | Promise<void>;
+    sourceEnv?: NodeJS.ProcessEnv;
+    platform?: NodeJS.Platform;
+    processTree?: import('./process-tree.js').ProcessTreeDependencies;
+}
+export interface RunExecutionResult {
+    valid: true;
+    bundlePath: string;
+    bundleDigest: Digest;
+    evidenceLevel: 'executed' | 'comparative';
+    requirementsMet: boolean;
+}
+export type SchemaName = 'aw-skill-eval-execution-plan' | 'eval-run-bundle' | 'eval-runtime-capabilities';

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "@lesliechan721/agent-skill-eval",
+  "version": "0.1.0-beta.1",
+  "type": "module",
+  "engines": {
+    "node": ">=22"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "bin": {
+    "aw-skill-eval": "./dist/bin.js"
+  },
+  "files": [
+    "dist/"
+  ],
+  "devDependencies": {
+    "@types/node": "^24.0.0",
+    "typescript": "^5.9.3",
+    "vitest": "^3.2.4"
+  },
+  "scripts": {
+    "build": "node scripts/clean-dist.mjs && tsc -p tsconfig.json && node scripts/embed-schemas.mjs",
+    "test": "pnpm run build && vitest run",
+    "test:unit": "pnpm run build && vitest run tests/unit"
+  }
+}